2024-11-14 09:47:32,115 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:47:32,130 main DEBUG Took 0.012463 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 09:47:32,131 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 09:47:32,131 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 09:47:32,132 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 09:47:32,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,143 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 09:47:32,155 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,157 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,158 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,159 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,161 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,161 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,162 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,162 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,163 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,164 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,165 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,165 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,166 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,167 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,168 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,168 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,169 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,170 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 09:47:32,171 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,172 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 09:47:32,174 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 09:47:32,175 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 09:47:32,178 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 09:47:32,179 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 09:47:32,180 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 09:47:32,182 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 09:47:32,191 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 09:47:32,194 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 09:47:32,196 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 09:47:32,197 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 09:47:32,197 main DEBUG createAppenders(={Console}) 2024-11-14 09:47:32,198 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 09:47:32,199 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 09:47:32,199 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 09:47:32,200 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 09:47:32,200 main DEBUG OutputStream closed 2024-11-14 09:47:32,200 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 09:47:32,201 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 09:47:32,201 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 09:47:32,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 09:47:32,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 09:47:32,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 09:47:32,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 09:47:32,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 09:47:32,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 09:47:32,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 09:47:32,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 09:47:32,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 09:47:32,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 09:47:32,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 09:47:32,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 09:47:32,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 09:47:32,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 09:47:32,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 09:47:32,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 09:47:32,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 09:47:32,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 09:47:32,312 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 09:47:32,313 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 09:47:32,313 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 09:47:32,314 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T09:47:32,577 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6 2024-11-14 09:47:32,580 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 09:47:32,581 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T09:47:32,589 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-14T09:47:32,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T09:47:32,637 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b, deleteOnExit=true 2024-11-14T09:47:32,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T09:47:32,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/test.cache.data in system properties and HBase conf 2024-11-14T09:47:32,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T09:47:32,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.log.dir in system properties and HBase conf 2024-11-14T09:47:32,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T09:47:32,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T09:47:32,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T09:47:32,733 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T09:47:32,829 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T09:47:32,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:47:32,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T09:47:32,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T09:47:32,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:47:32,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T09:47:32,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T09:47:32,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T09:47:32,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:47:32,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T09:47:32,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/nfs.dump.dir in system properties and HBase conf 2024-11-14T09:47:32,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/java.io.tmpdir in system properties and HBase conf 2024-11-14T09:47:32,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T09:47:32,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T09:47:32,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T09:47:34,032 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T09:47:34,105 INFO [Time-limited test {}] log.Log(170): Logging initialized @2831ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T09:47:34,174 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:47:34,238 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:47:34,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:47:34,262 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:47:34,263 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:47:34,274 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:47:34,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:47:34,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:47:34,486 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/java.io.tmpdir/jetty-localhost-35591-hadoop-hdfs-3_4_1-tests_jar-_-any-12727657807631458090/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:47:34,493 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35591} 2024-11-14T09:47:34,493 INFO [Time-limited test {}] server.Server(415): Started @3221ms 2024-11-14T09:47:35,054 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T09:47:35,061 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T09:47:35,062 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T09:47:35,063 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T09:47:35,063 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T09:47:35,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.log.dir/,AVAILABLE} 2024-11-14T09:47:35,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T09:47:35,164 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/java.io.tmpdir/jetty-localhost-38793-hadoop-hdfs-3_4_1-tests_jar-_-any-15668525872361312268/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:47:35,165 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:38793} 2024-11-14T09:47:35,165 INFO [Time-limited test {}] server.Server(415): Started @3892ms 2024-11-14T09:47:35,212 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T09:47:36,217 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/data/data2/current/BP-1058783131-172.17.0.2-1731577653437/current, will proceed with Du for space computation calculation, 2024-11-14T09:47:36,217 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/data/data1/current/BP-1058783131-172.17.0.2-1731577653437/current, will proceed with Du for space computation calculation, 2024-11-14T09:47:36,248 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T09:47:36,293 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x558070c6c7f5a98b with lease ID 0xc33075a5546de56f: Processing first storage report for DS-0a98365d-854c-4ee1-88dc-e9411b3cf75f from datanode DatanodeRegistration(127.0.0.1:44663, datanodeUuid=c8d1fb9d-fa1d-42ec-9043-d1b24d0eb8ba, infoPort=37495, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1271943629;c=1731577653437) 2024-11-14T09:47:36,294 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x558070c6c7f5a98b with lease ID 0xc33075a5546de56f: from storage DS-0a98365d-854c-4ee1-88dc-e9411b3cf75f node DatanodeRegistration(127.0.0.1:44663, datanodeUuid=c8d1fb9d-fa1d-42ec-9043-d1b24d0eb8ba, infoPort=37495, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1271943629;c=1731577653437), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T09:47:36,294 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x558070c6c7f5a98b with lease ID 0xc33075a5546de56f: Processing first storage report for DS-41b457c4-cbc0-44cf-aa83-543516a77a91 from datanode DatanodeRegistration(127.0.0.1:44663, datanodeUuid=c8d1fb9d-fa1d-42ec-9043-d1b24d0eb8ba, infoPort=37495, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1271943629;c=1731577653437) 2024-11-14T09:47:36,295 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x558070c6c7f5a98b with lease ID 0xc33075a5546de56f: from storage DS-41b457c4-cbc0-44cf-aa83-543516a77a91 node DatanodeRegistration(127.0.0.1:44663, datanodeUuid=c8d1fb9d-fa1d-42ec-9043-d1b24d0eb8ba, infoPort=37495, infoSecurePort=0, ipcPort=41403, storageInfo=lv=-57;cid=testClusterID;nsid=1271943629;c=1731577653437), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T09:47:36,379 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6 2024-11-14T09:47:36,452 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/zookeeper_0, clientPort=52200, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T09:47:36,461 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52200 2024-11-14T09:47:36,472 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:36,476 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741825_1001 (size=7) 2024-11-14T09:47:37,086 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 with version=8 2024-11-14T09:47:37,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/hbase-staging 2024-11-14T09:47:37,174 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T09:47:37,477 INFO [Time-limited test {}] client.ConnectionUtils(128): master/defc576eb6b7:0 server-side Connection retries=45 2024-11-14T09:47:37,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:37,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:37,494 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:47:37,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:37,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:47:37,623 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T09:47:37,672 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T09:47:37,679 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T09:47:37,683 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:47:37,705 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20408 (auto-detected) 2024-11-14T09:47:37,706 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T09:47:37,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35261 2024-11-14T09:47:37,743 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35261 connecting to ZooKeeper ensemble=127.0.0.1:52200 2024-11-14T09:47:37,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352610x0, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:47:37,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35261-0x10138be301c0000 connected 2024-11-14T09:47:37,961 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:37,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:37,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:47:37,978 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3, hbase.cluster.distributed=false 2024-11-14T09:47:38,004 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:47:38,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35261 2024-11-14T09:47:38,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35261 2024-11-14T09:47:38,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35261 2024-11-14T09:47:38,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35261 2024-11-14T09:47:38,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35261 2024-11-14T09:47:38,120 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/defc576eb6b7:0 server-side Connection retries=45 2024-11-14T09:47:38,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:38,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:38,122 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T09:47:38,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T09:47:38,123 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T09:47:38,125 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T09:47:38,128 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T09:47:38,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40243 2024-11-14T09:47:38,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40243 connecting to ZooKeeper ensemble=127.0.0.1:52200 2024-11-14T09:47:38,133 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:38,136 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:38,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402430x0, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T09:47:38,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:402430x0, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:47:38,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40243-0x10138be301c0001 connected 2024-11-14T09:47:38,158 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T09:47:38,167 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T09:47:38,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T09:47:38,175 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T09:47:38,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-14T09:47:38,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40243 2024-11-14T09:47:38,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40243 2024-11-14T09:47:38,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-14T09:47:38,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40243 2024-11-14T09:47:38,201 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;defc576eb6b7:35261 2024-11-14T09:47:38,203 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/defc576eb6b7,35261,1731577657311 2024-11-14T09:47:38,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:47:38,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:47:38,223 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/defc576eb6b7,35261,1731577657311 2024-11-14T09:47:38,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T09:47:38,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:38,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:38,254 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T09:47:38,255 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/defc576eb6b7,35261,1731577657311 from backup master directory 2024-11-14T09:47:38,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:47:38,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/defc576eb6b7,35261,1731577657311 2024-11-14T09:47:38,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T09:47:38,269 WARN [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:47:38,270 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=defc576eb6b7,35261,1731577657311 2024-11-14T09:47:38,272 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T09:47:38,273 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T09:47:38,326 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/hbase.id] with ID: e7792676-cf25-4a20-a379-7c54be1e39c6 2024-11-14T09:47:38,326 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/.tmp/hbase.id 2024-11-14T09:47:38,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741826_1002 (size=42) 2024-11-14T09:47:38,738 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/.tmp/hbase.id]:[hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/hbase.id] 2024-11-14T09:47:38,785 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T09:47:38,789 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T09:47:38,808 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-14T09:47:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:38,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:38,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741827_1003 (size=196) 2024-11-14T09:47:39,306 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:47:39,310 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T09:47:39,328 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:39,333 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:47:39,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741828_1004 (size=1189) 2024-11-14T09:47:39,782 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store 2024-11-14T09:47:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741829_1005 (size=34) 2024-11-14T09:47:40,213 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T09:47:40,216 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:40,217 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:47:40,217 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:47:40,217 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:47:40,219 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:47:40,219 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:47:40,219 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:47:40,220 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731577660217Disabling compacts and flushes for region at 1731577660217Disabling writes for close at 1731577660219 (+2 ms)Writing region close event to WAL at 1731577660219Closed at 1731577660219 2024-11-14T09:47:40,223 WARN [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/.initializing 2024-11-14T09:47:40,223 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/WALs/defc576eb6b7,35261,1731577657311 2024-11-14T09:47:40,232 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:47:40,246 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C35261%2C1731577657311, suffix=, logDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/WALs/defc576eb6b7,35261,1731577657311, archiveDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/oldWALs, maxLogs=10 2024-11-14T09:47:40,266 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/WALs/defc576eb6b7,35261,1731577657311/defc576eb6b7%2C35261%2C1731577657311.1731577660250, exclude list is [], retry=0 2024-11-14T09:47:40,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-0a98365d-854c-4ee1-88dc-e9411b3cf75f,DISK] 2024-11-14T09:47:40,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T09:47:40,315 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/WALs/defc576eb6b7,35261,1731577657311/defc576eb6b7%2C35261%2C1731577657311.1731577660250 2024-11-14T09:47:40,316 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37495:37495)] 2024-11-14T09:47:40,317 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:47:40,317 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:40,320 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,321 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T09:47:40,382 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:40,385 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:40,386 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T09:47:40,391 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:40,393 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:40,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T09:47:40,398 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:40,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:40,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T09:47:40,404 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:40,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:40,406 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,410 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,411 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,416 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,417 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,422 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T09:47:40,426 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T09:47:40,430 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:47:40,432 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70684879, jitterRate=0.053286775946617126}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T09:47:40,438 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731577660331Initializing all the Stores at 1731577660333 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577660334 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577660334Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577660335 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577660335Cleaning up temporary data from old regions at 1731577660417 (+82 ms)Region opened successfully at 1731577660438 (+21 ms) 2024-11-14T09:47:40,439 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T09:47:40,475 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d43f4bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=defc576eb6b7/172.17.0.2:0 2024-11-14T09:47:40,505 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T09:47:40,516 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T09:47:40,516 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T09:47:40,519 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T09:47:40,520 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T09:47:40,525 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T09:47:40,525 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T09:47:40,551 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T09:47:40,562 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T09:47:40,610 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T09:47:40,615 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T09:47:40,618 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T09:47:40,631 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T09:47:40,634 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T09:47:40,640 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T09:47:40,652 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T09:47:40,654 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T09:47:40,663 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T09:47:40,685 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T09:47:40,694 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T09:47:40,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:47:40,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T09:47:40,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,709 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=defc576eb6b7,35261,1731577657311, sessionid=0x10138be301c0000, setting cluster-up flag (Was=false) 2024-11-14T09:47:40,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,773 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T09:47:40,775 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=defc576eb6b7,35261,1731577657311 2024-11-14T09:47:40,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:40,831 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T09:47:40,835 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=defc576eb6b7,35261,1731577657311 2024-11-14T09:47:40,846 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T09:47:40,893 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(746): ClusterId : e7792676-cf25-4a20-a379-7c54be1e39c6 2024-11-14T09:47:40,895 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T09:47:40,907 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T09:47:40,907 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T09:47:40,917 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T09:47:40,917 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T09:47:40,917 DEBUG [RS:0;defc576eb6b7:40243 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c4ddb85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=defc576eb6b7/172.17.0.2:0 2024-11-14T09:47:40,926 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T09:47:40,934 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T09:47:40,934 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;defc576eb6b7:40243 2024-11-14T09:47:40,938 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T09:47:40,938 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T09:47:40,938 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T09:47:40,941 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(2659): reportForDuty to master=defc576eb6b7,35261,1731577657311 with port=40243, startcode=1731577658084 2024-11-14T09:47:40,939 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: defc576eb6b7,35261,1731577657311 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T09:47:40,946 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:47:40,946 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:47:40,946 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:47:40,947 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/defc576eb6b7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T09:47:40,947 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/defc576eb6b7:0, corePoolSize=10, maxPoolSize=10 2024-11-14T09:47:40,947 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:40,947 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/defc576eb6b7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:47:40,948 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:40,951 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731577690951 2024-11-14T09:47:40,953 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T09:47:40,954 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:47:40,954 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T09:47:40,954 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T09:47:40,954 DEBUG [RS:0;defc576eb6b7:40243 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T09:47:40,957 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T09:47:40,958 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T09:47:40,958 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T09:47:40,958 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T09:47:40,959 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:40,959 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T09:47:40,959 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:40,962 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T09:47:40,964 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T09:47:40,964 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T09:47:40,967 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T09:47:40,967 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T09:47:40,971 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577660968,5,FailOnTimeoutGroup] 2024-11-14T09:47:40,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741831_1007 (size=1321) 2024-11-14T09:47:40,972 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577660971,5,FailOnTimeoutGroup] 2024-11-14T09:47:40,973 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:40,973 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T09:47:40,974 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:40,974 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:40,992 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T09:47:40,998 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35261 {}] master.ServerManager(363): Checking decommissioned status of RegionServer defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,000 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35261 {}] master.ServerManager(517): Registering regionserver=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,012 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:47:41,012 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44807 2024-11-14T09:47:41,012 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T09:47:41,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:47:41,026 DEBUG [RS:0;defc576eb6b7:40243 {}] zookeeper.ZKUtil(111): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,027 WARN [RS:0;defc576eb6b7:40243 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T09:47:41,027 INFO [RS:0;defc576eb6b7:40243 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:47:41,027 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,029 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [defc576eb6b7,40243,1731577658084] 2024-11-14T09:47:41,050 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T09:47:41,065 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T09:47:41,070 INFO [RS:0;defc576eb6b7:40243 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T09:47:41,070 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,071 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T09:47:41,076 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T09:47:41,078 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,078 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,078 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,078 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/defc576eb6b7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,079 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,080 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,080 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/defc576eb6b7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T09:47:41,080 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/defc576eb6b7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:47:41,080 DEBUG [RS:0;defc576eb6b7:40243 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T09:47:41,081 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,082 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,082 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,082 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,082 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,082 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,40243,1731577658084-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:47:41,101 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T09:47:41,103 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,40243,1731577658084-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,103 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,104 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.Replication(171): defc576eb6b7,40243,1731577658084 started 2024-11-14T09:47:41,125 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:41,125 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1482): Serving as defc576eb6b7,40243,1731577658084, RpcServer on defc576eb6b7/172.17.0.2:40243, sessionid=0x10138be301c0001 2024-11-14T09:47:41,126 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T09:47:41,126 DEBUG [RS:0;defc576eb6b7:40243 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,127 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'defc576eb6b7,40243,1731577658084' 2024-11-14T09:47:41,127 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T09:47:41,128 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T09:47:41,128 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T09:47:41,128 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T09:47:41,129 DEBUG [RS:0;defc576eb6b7:40243 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager defc576eb6b7,40243,1731577658084 2024-11-14T09:47:41,129 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'defc576eb6b7,40243,1731577658084' 2024-11-14T09:47:41,129 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T09:47:41,129 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T09:47:41,130 DEBUG [RS:0;defc576eb6b7:40243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T09:47:41,130 INFO [RS:0;defc576eb6b7:40243 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T09:47:41,130 INFO [RS:0;defc576eb6b7:40243 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T09:47:41,234 INFO [RS:0;defc576eb6b7:40243 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T09:47:41,237 INFO [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C40243%2C1731577658084, suffix=, logDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084, archiveDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/oldWALs, maxLogs=32 2024-11-14T09:47:41,253 DEBUG [RS:0;defc576eb6b7:40243 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084/defc576eb6b7%2C40243%2C1731577658084.1731577661240, exclude list is [], retry=0 2024-11-14T09:47:41,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-0a98365d-854c-4ee1-88dc-e9411b3cf75f,DISK] 2024-11-14T09:47:41,262 INFO [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084/defc576eb6b7%2C40243%2C1731577658084.1731577661240 2024-11-14T09:47:41,262 DEBUG [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37495:37495)] 2024-11-14T09:47:41,375 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T09:47:41,376 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:47:41,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741833_1009 (size=32) 2024-11-14T09:47:41,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:41,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:47:41,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:47:41,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:41,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:41,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:47:41,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:47:41,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:41,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:41,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:47:41,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:47:41,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:41,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:41,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:47:41,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:47:41,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:41,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:41,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:47:41,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740 2024-11-14T09:47:41,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740 2024-11-14T09:47:41,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:47:41,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:47:41,819 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:47:41,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:47:41,825 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:47:41,826 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69912630, jitterRate=0.04177936911582947}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:47:41,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731577661788Initializing all the Stores at 1731577661790 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577661790Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577661790Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577661790Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577661790Cleaning up temporary data from old regions at 1731577661817 (+27 ms)Region opened successfully at 1731577661828 (+11 ms) 2024-11-14T09:47:41,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:47:41,828 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:47:41,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:47:41,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:47:41,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:47:41,830 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:47:41,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731577661828Disabling compacts and flushes for region at 1731577661828Disabling writes for close at 1731577661829 (+1 ms)Writing region close event to WAL at 1731577661829Closed at 1731577661830 (+1 ms) 2024-11-14T09:47:41,833 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:47:41,833 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T09:47:41,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T09:47:41,845 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:47:41,848 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T09:47:42,003 DEBUG [defc576eb6b7:35261 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T09:47:42,018 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:42,025 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as defc576eb6b7,40243,1731577658084, state=OPENING 2024-11-14T09:47:42,073 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T09:47:42,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:42,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:47:42,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:47:42,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:47:42,086 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T09:47:42,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:47:42,259 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T09:47:42,262 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T09:47:42,273 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T09:47:42,273 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T09:47:42,274 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T09:47:42,277 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=defc576eb6b7%2C40243%2C1731577658084.meta, suffix=.meta, logDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084, archiveDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/oldWALs, maxLogs=32 2024-11-14T09:47:42,297 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084/defc576eb6b7%2C40243%2C1731577658084.meta.1731577662280.meta, exclude list is [], retry=0 2024-11-14T09:47:42,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44663,DS-0a98365d-854c-4ee1-88dc-e9411b3cf75f,DISK] 2024-11-14T09:47:42,304 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/WALs/defc576eb6b7,40243,1731577658084/defc576eb6b7%2C40243%2C1731577658084.meta.1731577662280.meta 2024-11-14T09:47:42,304 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37495:37495)] 2024-11-14T09:47:42,305 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:47:42,307 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T09:47:42,310 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T09:47:42,316 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T09:47:42,320 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T09:47:42,322 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:42,322 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T09:47:42,322 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T09:47:42,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T09:47:42,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T09:47:42,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:42,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:42,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T09:47:42,331 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T09:47:42,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:42,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:42,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T09:47:42,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T09:47:42,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:42,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:42,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T09:47:42,338 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T09:47:42,338 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:42,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T09:47:42,339 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T09:47:42,340 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740 2024-11-14T09:47:42,342 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740 2024-11-14T09:47:42,345 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T09:47:42,345 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T09:47:42,346 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:47:42,348 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T09:47:42,350 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66412558, jitterRate=-0.010375767946243286}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:47:42,350 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T09:47:42,351 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731577662323Writing region info on filesystem at 1731577662323Initializing all the Stores at 1731577662325 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577662325Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577662325Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577662325Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731577662325Cleaning up temporary data from old regions at 1731577662345 (+20 ms)Running coprocessor post-open hooks at 1731577662350 (+5 ms)Region opened successfully at 1731577662351 (+1 ms) 2024-11-14T09:47:42,357 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731577662251 2024-11-14T09:47:42,368 DEBUG [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T09:47:42,368 INFO [RS_OPEN_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T09:47:42,369 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:42,372 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as defc576eb6b7,40243,1731577658084, state=OPEN 2024-11-14T09:47:42,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:47:42,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T09:47:42,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:47:42,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T09:47:42,498 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:42,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T09:47:42,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=defc576eb6b7,40243,1731577658084 in 411 msec 2024-11-14T09:47:42,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T09:47:42,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 667 msec 2024-11-14T09:47:42,512 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T09:47:42,512 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T09:47:42,529 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:42,531 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:42,556 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:42,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57333, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:42,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7020 sec 2024-11-14T09:47:42,578 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731577662578, completionTime=-1 2024-11-14T09:47:42,581 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T09:47:42,581 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T09:47:42,605 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T09:47:42,605 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731577722605 2024-11-14T09:47:42,605 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731577782605 2024-11-14T09:47:42,606 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-14T09:47:42,608 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,608 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,608 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,610 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-defc576eb6b7:35261, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,610 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,611 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,616 DEBUG [master/defc576eb6b7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T09:47:42,638 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.368sec 2024-11-14T09:47:42,640 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T09:47:42,642 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T09:47:42,643 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T09:47:42,644 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T09:47:42,644 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T09:47:42,644 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T09:47:42,645 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T09:47:42,652 DEBUG [master/defc576eb6b7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T09:47:42,653 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T09:47:42,653 INFO [master/defc576eb6b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=defc576eb6b7,35261,1731577657311-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T09:47:42,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:42,709 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T09:47:42,709 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T09:47:42,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:42,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:42,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:42,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:42,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2479a655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:42,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:42,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:42,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:42,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:42,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:42,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:42,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=defc576eb6b7,35261,1731577657311 2024-11-14T09:47:42,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=5082 2024-11-14T09:47:42,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:47:42,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:47:42,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c1af104 2024-11-14T09:47:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:47:42,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:47:42,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:47:42,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:47:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:47:42,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:47:42,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T09:47:42,858 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:42,861 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:47:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:42,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741835_1011 (size=959) 2024-11-14T09:47:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:43,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:43,288 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:47:43,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741836_1012 (size=53) 2024-11-14T09:47:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:43,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:43,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 91a5f3dba40c0b5503a8f1cccd94111e, disabling compactions & flushes 2024-11-14T09:47:43,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:43,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:43,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. after waiting 0 ms 2024-11-14T09:47:43,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:43,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:43,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 91a5f3dba40c0b5503a8f1cccd94111e: Waiting for close lock at 1731577663698Disabling compacts and flushes for region at 1731577663698Disabling writes for close at 1731577663698Writing region close event to WAL at 1731577663698Closed at 1731577663698 2024-11-14T09:47:43,701 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:47:43,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577663701"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577663701"}]},"ts":"1731577663701"} 2024-11-14T09:47:43,710 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:47:43,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:47:43,714 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577663712"}]},"ts":"1731577663712"} 2024-11-14T09:47:43,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:47:43,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, ASSIGN}] 2024-11-14T09:47:43,723 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, ASSIGN 2024-11-14T09:47:43,724 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:47:43,876 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=91a5f3dba40c0b5503a8f1cccd94111e, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:43,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, ASSIGN because future has completed 2024-11-14T09:47:43,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:47:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:44,043 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:44,044 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:47:44,044 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,044 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:47:44,045 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,045 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,047 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,050 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a5f3dba40c0b5503a8f1cccd94111e columnFamilyName A 2024-11-14T09:47:44,050 DEBUG [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:44,051 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(327): Store=91a5f3dba40c0b5503a8f1cccd94111e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:44,051 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,054 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a5f3dba40c0b5503a8f1cccd94111e columnFamilyName B 2024-11-14T09:47:44,054 DEBUG [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:44,055 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(327): Store=91a5f3dba40c0b5503a8f1cccd94111e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:44,055 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,058 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91a5f3dba40c0b5503a8f1cccd94111e columnFamilyName C 2024-11-14T09:47:44,058 DEBUG [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:47:44,059 INFO [StoreOpener-91a5f3dba40c0b5503a8f1cccd94111e-1 {}] regionserver.HStore(327): Store=91a5f3dba40c0b5503a8f1cccd94111e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:47:44,060 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,061 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,062 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,065 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,065 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,066 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:47:44,069 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,074 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:47:44,075 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 91a5f3dba40c0b5503a8f1cccd94111e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67109435, jitterRate=8.508563041687012E-6}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:47:44,075 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:44,077 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 91a5f3dba40c0b5503a8f1cccd94111e: Running coprocessor pre-open hook at 1731577664045Writing region info on filesystem at 1731577664045Initializing all the Stores at 1731577664047 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577664047Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577664047Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577664047Cleaning up temporary data from old regions at 1731577664065 (+18 ms)Running coprocessor post-open hooks at 1731577664075 (+10 ms)Region opened successfully at 1731577664077 (+2 ms) 2024-11-14T09:47:44,079 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., pid=6, masterSystemTime=1731577664037 2024-11-14T09:47:44,083 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:44,083 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:44,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=91a5f3dba40c0b5503a8f1cccd94111e, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:44,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:47:44,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T09:47:44,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 in 209 msec 2024-11-14T09:47:44,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T09:47:44,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, ASSIGN in 377 msec 2024-11-14T09:47:44,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:47:44,104 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577664104"}]},"ts":"1731577664104"} 2024-11-14T09:47:44,107 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:47:44,109 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:47:44,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2640 sec 2024-11-14T09:47:45,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T09:47:45,016 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:45,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,026 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665358ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,028 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,032 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,034 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:47:45,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:47:45,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b26fa3 2024-11-14T09:47:45,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:47:45,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:47:45,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,047 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,047 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,048 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,055 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,060 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,063 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,063 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,066 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,072 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,072 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,072 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,074 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,078 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbd72e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,080 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,080 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,082 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6af04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,085 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5eb702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,090 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0b2d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c270ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec64ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e58d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,103 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c24f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,106 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:47:45,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:47:45,110 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:47:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:47:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:47:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:47:45,111 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:47:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:47:45,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:47:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:47:45,119 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,119 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,127 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,141 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,158 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,162 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,172 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,174 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,179 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,179 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,190 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:47:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:47:45,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:47:45,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:47:45,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:47:45,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T09:47:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:47:45,233 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:45,240 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:45,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:45,308 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:45,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T09:47:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:47:45,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:47:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:45,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577725429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/3920a6b757d849fcbd2806d16648fc92 is 50, key is test_row_0/A:col10/1731577665262/Put/seqid=0 2024-11-14T09:47:45,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577725446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577725451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577725452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577725440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741837_1013 (size=12001) 2024-11-14T09:47:45,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/3920a6b757d849fcbd2806d16648fc92 2024-11-14T09:47:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:47:45,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:47:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577725616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577725617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577725618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577725618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:45,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577725619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3806a45f9ca740ca9d24ddcdd3da5a2a is 50, key is test_row_0/B:col10/1731577665262/Put/seqid=0 2024-11-14T09:47:45,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741838_1014 (size=12001) 2024-11-14T09:47:45,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3806a45f9ca740ca9d24ddcdd3da5a2a 2024-11-14T09:47:45,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/20dd02cb058c4631ba267a902bba00f3 is 50, key is test_row_0/C:col10/1731577665262/Put/seqid=0 2024-11-14T09:47:45,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:47:45,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741839_1015 (size=12001) 2024-11-14T09:47:45,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/20dd02cb058c4631ba267a902bba00f3 2024-11-14T09:47:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:45,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/3920a6b757d849fcbd2806d16648fc92 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92 2024-11-14T09:47:45,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:47:45,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577725835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3806a45f9ca740ca9d24ddcdd3da5a2a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a 2024-11-14T09:47:45,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577725835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577725836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577725837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577725837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:45,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:47:45,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:47:45,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/20dd02cb058c4631ba267a902bba00f3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3 2024-11-14T09:47:45,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:47:45,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 91a5f3dba40c0b5503a8f1cccd94111e in 605ms, sequenceid=14, compaction requested=false 2024-11-14T09:47:45,923 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:47:45,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:45,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T09:47:45,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:45,968 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T09:47:45,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/e40f1c3322454ec9984260f5aa203bf8 is 50, key is test_row_0/A:col10/1731577665441/Put/seqid=0 2024-11-14T09:47:45,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741840_1016 (size=12001) 2024-11-14T09:47:46,000 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/e40f1c3322454ec9984260f5aa203bf8 2024-11-14T09:47:46,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c81347d2da344cf79c9a0b63c5d8e734 is 50, key is test_row_0/B:col10/1731577665441/Put/seqid=0 2024-11-14T09:47:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741841_1017 (size=12001) 2024-11-14T09:47:46,071 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c81347d2da344cf79c9a0b63c5d8e734 2024-11-14T09:47:46,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/f7166ea9637d486cb6294bbd4815249f is 50, key is test_row_0/C:col10/1731577665441/Put/seqid=0 2024-11-14T09:47:46,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741842_1018 (size=12001) 2024-11-14T09:47:46,139 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/f7166ea9637d486cb6294bbd4815249f 2024-11-14T09:47:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/e40f1c3322454ec9984260f5aa203bf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8 2024-11-14T09:47:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:46,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:46,181 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:47:46,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c81347d2da344cf79c9a0b63c5d8e734 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734 2024-11-14T09:47:46,202 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:47:46,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/f7166ea9637d486cb6294bbd4815249f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f 2024-11-14T09:47:46,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577726192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577726198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577726195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,223 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:47:46,227 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 91a5f3dba40c0b5503a8f1cccd94111e in 258ms, sequenceid=37, compaction requested=false 2024-11-14T09:47:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:46,228 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:46,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:47:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T09:47:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T09:47:46,238 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T09:47:46,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-14T09:47:46,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/77ab293cffe046d39d00ab6fb15e0182 is 50, key is test_row_0/A:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:46,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0200 sec 2024-11-14T09:47:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741843_1019 (size=14341) 2024-11-14T09:47:46,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/77ab293cffe046d39d00ab6fb15e0182 2024-11-14T09:47:46,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6461a7545ff145fabed8199a109d02b7 is 50, key is test_row_0/B:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741844_1020 (size=12001) 2024-11-14T09:47:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577726341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577726350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T09:47:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577726356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,366 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577726358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577726359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T09:47:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:46,371 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:46,373 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:46,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:46,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577726465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:46,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577726477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577726477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577726479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577726479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577726675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:46,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577726686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577726695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577726696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577726696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:46,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6461a7545ff145fabed8199a109d02b7 2024-11-14T09:47:46,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5ceb90bcd71042dfb65921afaece8ffe is 50, key is test_row_0/C:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:46,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741845_1021 (size=12001) 2024-11-14T09:47:46,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5ceb90bcd71042dfb65921afaece8ffe 2024-11-14T09:47:46,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/77ab293cffe046d39d00ab6fb15e0182 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182 2024-11-14T09:47:46,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182, entries=200, sequenceid=52, filesize=14.0 K 2024-11-14T09:47:46,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6461a7545ff145fabed8199a109d02b7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7 2024-11-14T09:47:46,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T09:47:46,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5ceb90bcd71042dfb65921afaece8ffe as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe 2024-11-14T09:47:46,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:46,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T09:47:46,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 91a5f3dba40c0b5503a8f1cccd94111e in 626ms, sequenceid=52, compaction requested=true 2024-11-14T09:47:46,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:46,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:46,870 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:46,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:46,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:46,871 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:46,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:46,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:46,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:46,874 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:46,876 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:46,877 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,878 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=35.2 K 2024-11-14T09:47:46,879 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:46,880 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:46,880 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,880 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.4 K 2024-11-14T09:47:46,881 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3920a6b757d849fcbd2806d16648fc92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577665227 2024-11-14T09:47:46,883 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3806a45f9ca740ca9d24ddcdd3da5a2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577665227 2024-11-14T09:47:46,884 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e40f1c3322454ec9984260f5aa203bf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577665404 2024-11-14T09:47:46,884 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c81347d2da344cf79c9a0b63c5d8e734, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577665404 2024-11-14T09:47:46,885 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6461a7545ff145fabed8199a109d02b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:46,888 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77ab293cffe046d39d00ab6fb15e0182, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:46,932 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:46,933 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ee85ac5dd55e4aec8b95371bcea61810 is 50, key is test_row_0/A:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:46,933 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:46,934 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/bca8113d2bd64094a54e2b9504c7e4e7 is 50, key is test_row_0/B:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:46,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741846_1022 (size=12104) 2024-11-14T09:47:46,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741847_1023 (size=12104) 2024-11-14T09:47:46,990 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/bca8113d2bd64094a54e2b9504c7e4e7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bca8113d2bd64094a54e2b9504c7e4e7 2024-11-14T09:47:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:46,992 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:46,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T09:47:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:46,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:46,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:47,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/584970e36f194727b0183f0e488265cb is 50, key is test_row_0/A:col10/1731577666348/Put/seqid=0 2024-11-14T09:47:47,025 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into bca8113d2bd64094a54e2b9504c7e4e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:47,025 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:47,028 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577666871; duration=0sec 2024-11-14T09:47:47,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741848_1024 (size=14341) 2024-11-14T09:47:47,029 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:47,030 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:47,031 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:47,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/584970e36f194727b0183f0e488265cb 2024-11-14T09:47:47,035 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:47,035 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:47,035 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:47,036 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=35.2 K 2024-11-14T09:47:47,037 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 20dd02cb058c4631ba267a902bba00f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577665227 2024-11-14T09:47:47,039 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f7166ea9637d486cb6294bbd4815249f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577665404 2024-11-14T09:47:47,042 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ceb90bcd71042dfb65921afaece8ffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:47,045 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T09:47:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577727033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577727042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6d8d59b765cb479e8f708a4b3c2ac4ca is 50, key is test_row_0/B:col10/1731577666348/Put/seqid=0 2024-11-14T09:47:47,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577727054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577727058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,077 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#13 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:47,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,078 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/a5338eafed3d4cf2a7845ee3837f8b4e is 50, key is test_row_0/C:col10/1731577666186/Put/seqid=0 2024-11-14T09:47:47,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577727077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741849_1025 (size=12001) 2024-11-14T09:47:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741850_1026 (size=12104) 2024-11-14T09:47:47,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6d8d59b765cb479e8f708a4b3c2ac4ca 2024-11-14T09:47:47,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/0c5773ce20ed4316ab01791f39b09e12 is 50, key is test_row_0/C:col10/1731577666348/Put/seqid=0 2024-11-14T09:47:47,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741851_1027 (size=12001) 2024-11-14T09:47:47,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/0c5773ce20ed4316ab01791f39b09e12 2024-11-14T09:47:47,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:47,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:47,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/584970e36f194727b0183f0e488265cb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb 2024-11-14T09:47:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577727173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577727174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb, entries=200, sequenceid=75, filesize=14.0 K 2024-11-14T09:47:47,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/6d8d59b765cb479e8f708a4b3c2ac4ca as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca 2024-11-14T09:47:47,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577727185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577727196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577727196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T09:47:47,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/0c5773ce20ed4316ab01791f39b09e12 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12 2024-11-14T09:47:47,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T09:47:47,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 91a5f3dba40c0b5503a8f1cccd94111e in 238ms, sequenceid=75, compaction requested=false 2024-11-14T09:47:47,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:47,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T09:47:47,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:47,313 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:47:47,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b6b146fd09354dd69c4dc00fc149645c is 50, key is test_row_0/A:col10/1731577667037/Put/seqid=0 2024-11-14T09:47:47,373 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ee85ac5dd55e4aec8b95371bcea61810 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ee85ac5dd55e4aec8b95371bcea61810 2024-11-14T09:47:47,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741852_1028 (size=12001) 2024-11-14T09:47:47,380 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b6b146fd09354dd69c4dc00fc149645c 2024-11-14T09:47:47,391 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into ee85ac5dd55e4aec8b95371bcea61810(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:47,391 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:47,391 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577666860; duration=0sec 2024-11-14T09:47:47,391 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:47,392 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:47,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/1f737f140afc44d1b5ba4bbe0d93476c is 50, key is test_row_0/B:col10/1731577667037/Put/seqid=0 2024-11-14T09:47:47,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741853_1029 (size=12001) 2024-11-14T09:47:47,494 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/1f737f140afc44d1b5ba4bbe0d93476c 2024-11-14T09:47:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:47,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577727488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577727489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577727489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,511 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/a5338eafed3d4cf2a7845ee3837f8b4e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/a5338eafed3d4cf2a7845ee3837f8b4e 2024-11-14T09:47:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577727493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577727499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/58ab3eec191e40108596acdef52bace1 is 50, key is test_row_0/C:col10/1731577667037/Put/seqid=0 2024-11-14T09:47:47,530 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into a5338eafed3d4cf2a7845ee3837f8b4e(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:47,530 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:47,530 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577666871; duration=0sec 2024-11-14T09:47:47,531 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:47,531 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:47,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741854_1030 (size=12001) 2024-11-14T09:47:47,537 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/58ab3eec191e40108596acdef52bace1 2024-11-14T09:47:47,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b6b146fd09354dd69c4dc00fc149645c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c 2024-11-14T09:47:47,572 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-14T09:47:47,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/1f737f140afc44d1b5ba4bbe0d93476c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c 2024-11-14T09:47:47,592 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-14T09:47:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/58ab3eec191e40108596acdef52bace1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1 2024-11-14T09:47:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577727615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577727615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577727615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577727616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,628 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-14T09:47:47,630 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 91a5f3dba40c0b5503a8f1cccd94111e in 318ms, sequenceid=90, compaction requested=true 2024-11-14T09:47:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T09:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T09:47:47,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:47,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:47,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T09:47:47,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2620 sec 2024-11-14T09:47:47,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/aa088fe3249d42eb90c8f543a9df54e2 is 50, key is test_row_0/A:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:47,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2780 sec 2024-11-14T09:47:47,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741855_1031 (size=12001) 2024-11-14T09:47:47,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577727719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577727825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577727825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:47:47,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577727826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577727835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:47,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577727835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:47,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:47,928 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T09:47:47,928 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T09:47:47,931 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:47:47,931 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T09:47:47,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T09:47:47,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T09:47:47,937 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T09:47:47,937 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T09:47:48,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577728055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/aa088fe3249d42eb90c8f543a9df54e2 2024-11-14T09:47:48,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/db8495a616be4682936d46f4b185155f is 50, key is test_row_0/B:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:48,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741856_1032 (size=12001) 2024-11-14T09:47:48,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/db8495a616be4682936d46f4b185155f 2024-11-14T09:47:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577728135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577728135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577728144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/7878169c210b4916880a819d2217cf1f is 50, key is test_row_0/C:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577728155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741857_1033 (size=12001) 2024-11-14T09:47:48,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577728365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T09:47:48,529 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:48,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T09:47:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:48,534 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:48,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:48,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:48,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/7878169c210b4916880a819d2217cf1f 2024-11-14T09:47:48,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/aa088fe3249d42eb90c8f543a9df54e2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2 2024-11-14T09:47:48,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:47:48,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/db8495a616be4682936d46f4b185155f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f 2024-11-14T09:47:48,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:47:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:48,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/7878169c210b4916880a819d2217cf1f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f 2024-11-14T09:47:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577728645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:47:48,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 91a5f3dba40c0b5503a8f1cccd94111e in 1021ms, sequenceid=116, compaction requested=true 2024-11-14T09:47:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:48,654 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:48,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:48,655 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:48,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:48,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:48,660 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:48,660 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:48,660 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,660 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:48,660 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:48,660 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,660 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bca8113d2bd64094a54e2b9504c7e4e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=47.0 K 2024-11-14T09:47:48,660 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ee85ac5dd55e4aec8b95371bcea61810, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=49.3 K 2024-11-14T09:47:48,661 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee85ac5dd55e4aec8b95371bcea61810, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:48,661 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting bca8113d2bd64094a54e2b9504c7e4e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:48,662 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 584970e36f194727b0183f0e488265cb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577666343 2024-11-14T09:47:48,662 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8d59b765cb479e8f708a4b3c2ac4ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577666343 2024-11-14T09:47:48,663 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6b146fd09354dd69c4dc00fc149645c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731577667037 2024-11-14T09:47:48,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f737f140afc44d1b5ba4bbe0d93476c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731577667037 2024-11-14T09:47:48,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:48,664 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting db8495a616be4682936d46f4b185155f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:48,665 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa088fe3249d42eb90c8f543a9df54e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:48,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:48,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T09:47:48,696 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:48,698 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/57452e2edff945768ecf160b06ad7bd8 is 50, key is test_row_0/A:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:48,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,708 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:48,709 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ad1055ece1e8416fb9760cd64046b928 is 50, key is test_row_0/B:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:48,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c42e266c879d4081b85818321fde1139 is 50, key is test_row_0/A:col10/1731577668684/Put/seqid=0 2024-11-14T09:47:48,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741858_1034 (size=12241) 2024-11-14T09:47:48,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577728752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577728756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577728758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741860_1036 (size=12101) 2024-11-14T09:47:48,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741859_1035 (size=12241) 2024-11-14T09:47:48,801 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ad1055ece1e8416fb9760cd64046b928 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ad1055ece1e8416fb9760cd64046b928 2024-11-14T09:47:48,814 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into ad1055ece1e8416fb9760cd64046b928(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:48,814 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:48,814 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=12, startTime=1731577668655; duration=0sec 2024-11-14T09:47:48,815 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:48,815 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:48,815 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:48,818 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:48,818 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:48,818 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,819 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/a5338eafed3d4cf2a7845ee3837f8b4e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=47.0 K 2024-11-14T09:47:48,820 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a5338eafed3d4cf2a7845ee3837f8b4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731577666186 2024-11-14T09:47:48,821 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5773ce20ed4316ab01791f39b09e12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577666343 2024-11-14T09:47:48,822 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 58ab3eec191e40108596acdef52bace1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731577667037 2024-11-14T09:47:48,823 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7878169c210b4916880a819d2217cf1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:48,856 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:48,857 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/993d19ae5dbe483a8267dab180b0f81f is 50, key is test_row_0/C:col10/1731577667449/Put/seqid=0 2024-11-14T09:47:48,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:48,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:48,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:48,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577728865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577728876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577728876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577728876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:48,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741861_1037 (size=12241) 2024-11-14T09:47:49,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577729075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577729085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577729086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,149 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/57452e2edff945768ecf160b06ad7bd8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/57452e2edff945768ecf160b06ad7bd8 2024-11-14T09:47:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:49,165 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into 57452e2edff945768ecf160b06ad7bd8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:49,165 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:49,165 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=12, startTime=1731577668654; duration=0sec 2024-11-14T09:47:49,165 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:49,165 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:49,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c42e266c879d4081b85818321fde1139 2024-11-14T09:47:49,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/a12f1e4631584c29ad9fd36cafa9feee is 50, key is test_row_0/B:col10/1731577668684/Put/seqid=0 2024-11-14T09:47:49,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741862_1038 (size=12101) 2024-11-14T09:47:49,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/a12f1e4631584c29ad9fd36cafa9feee 2024-11-14T09:47:49,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/431ca23b990e436ab6c678184431d6e0 is 50, key is test_row_0/C:col10/1731577668684/Put/seqid=0 2024-11-14T09:47:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741863_1039 (size=12101) 2024-11-14T09:47:49,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/431ca23b990e436ab6c678184431d6e0 2024-11-14T09:47:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c42e266c879d4081b85818321fde1139 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139 2024-11-14T09:47:49,316 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/993d19ae5dbe483a8267dab180b0f81f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/993d19ae5dbe483a8267dab180b0f81f 2024-11-14T09:47:49,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T09:47:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/a12f1e4631584c29ad9fd36cafa9feee as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee 2024-11-14T09:47:49,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,330 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 993d19ae5dbe483a8267dab180b0f81f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:49,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:49,330 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=12, startTime=1731577668659; duration=0sec 2024-11-14T09:47:49,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:49,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:49,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T09:47:49,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/431ca23b990e436ab6c678184431d6e0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0 2024-11-14T09:47:49,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0, entries=150, sequenceid=134, filesize=11.8 K 2024-11-14T09:47:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 91a5f3dba40c0b5503a8f1cccd94111e in 676ms, sequenceid=134, compaction requested=false 2024-11-14T09:47:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:49,398 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:49,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:47:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:49,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f383d1b041e849bdbce02a2dd2579a8e is 50, key is test_row_0/A:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577729443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577729443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577729445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741864_1040 (size=14541) 2024-11-14T09:47:49,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f383d1b041e849bdbce02a2dd2579a8e 2024-11-14T09:47:49,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ee1569e6e1fd42749b7120b80bf226d9 is 50, key is test_row_0/B:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741865_1041 (size=12151) 2024-11-14T09:47:49,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ee1569e6e1fd42749b7120b80bf226d9 2024-11-14T09:47:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577729556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577729565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577729565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/3d6306fbd76645858429783470ea7644 is 50, key is test_row_0/C:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741866_1042 (size=12151) 2024-11-14T09:47:49,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/3d6306fbd76645858429783470ea7644 2024-11-14T09:47:49,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f383d1b041e849bdbce02a2dd2579a8e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e 2024-11-14T09:47:49,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e, entries=200, sequenceid=156, filesize=14.2 K 2024-11-14T09:47:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ee1569e6e1fd42749b7120b80bf226d9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9 2024-11-14T09:47:49,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:47:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/3d6306fbd76645858429783470ea7644 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644 2024-11-14T09:47:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:47:49,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 91a5f3dba40c0b5503a8f1cccd94111e in 260ms, sequenceid=156, compaction requested=true 2024-11-14T09:47:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:49,659 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:49,661 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38883 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:49,661 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:49,662 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,662 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/57452e2edff945768ecf160b06ad7bd8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=38.0 K 2024-11-14T09:47:49,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:49,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:49,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:49,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:49,663 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57452e2edff945768ecf160b06ad7bd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:49,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:47:49,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:49,668 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42e266c879d4081b85818321fde1139, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731577667695 2024-11-14T09:47:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:49,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T09:47:49,673 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:49,673 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:49,673 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,673 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ad1055ece1e8416fb9760cd64046b928, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=35.6 K 2024-11-14T09:47:49,674 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f383d1b041e849bdbce02a2dd2579a8e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:49,679 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1055ece1e8416fb9760cd64046b928, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:49,682 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a12f1e4631584c29ad9fd36cafa9feee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731577667695 2024-11-14T09:47:49,683 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1569e6e1fd42749b7120b80bf226d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:49,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/495e8a8358be4be8b118551a4755700a is 50, key is test_row_0/A:col10/1731577669437/Put/seqid=0 2024-11-14T09:47:49,724 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:49,725 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/cb1034fc6a5443f19a79b24160ef7d8f is 50, key is test_row_0/A:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,746 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#32 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:49,749 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/09fd49baf8ba40a290cdcd4c3483db17 is 50, key is test_row_0/B:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741868_1044 (size=12493) 2024-11-14T09:47:49,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741867_1043 (size=14541) 2024-11-14T09:47:49,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741869_1045 (size=12493) 2024-11-14T09:47:49,775 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/cb1034fc6a5443f19a79b24160ef7d8f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cb1034fc6a5443f19a79b24160ef7d8f 2024-11-14T09:47:49,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577729777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577729777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577729778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577729779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,794 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into cb1034fc6a5443f19a79b24160ef7d8f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:49,794 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:49,795 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577669659; duration=0sec 2024-11-14T09:47:49,796 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:49,796 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:49,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,796 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:49,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,798 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:49,799 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:49,799 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,800 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/993d19ae5dbe483a8267dab180b0f81f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=35.6 K 2024-11-14T09:47:49,801 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 993d19ae5dbe483a8267dab180b0f81f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577667449 2024-11-14T09:47:49,802 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 431ca23b990e436ab6c678184431d6e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731577667695 2024-11-14T09:47:49,803 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d6306fbd76645858429783470ea7644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:49,832 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:49,834 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d16abc56cf8b40bfb93be9fd3eee5c7e is 50, key is test_row_0/C:col10/1731577668748/Put/seqid=0 2024-11-14T09:47:49,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741870_1046 (size=12493) 2024-11-14T09:47:49,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577729895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577729895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577729895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,902 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:49,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:49,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:49,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577730085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577730096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:50,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577730105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577730105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/495e8a8358be4be8b118551a4755700a 2024-11-14T09:47:50,180 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/09fd49baf8ba40a290cdcd4c3483db17 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09fd49baf8ba40a290cdcd4c3483db17 2024-11-14T09:47:50,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/75f2d5d80b78443f8351b2bcbf5242b3 is 50, key is test_row_0/B:col10/1731577669437/Put/seqid=0 2024-11-14T09:47:50,195 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 09fd49baf8ba40a290cdcd4c3483db17(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:50,195 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:50,195 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577669663; duration=0sec 2024-11-14T09:47:50,196 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:50,196 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:50,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741871_1047 (size=12151) 2024-11-14T09:47:50,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/75f2d5d80b78443f8351b2bcbf5242b3 2024-11-14T09:47:50,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c2f4ee078a2249b3b57b25a464bf7a37 is 50, key is test_row_0/C:col10/1731577669437/Put/seqid=0 2024-11-14T09:47:50,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741872_1048 (size=12151) 2024-11-14T09:47:50,269 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d16abc56cf8b40bfb93be9fd3eee5c7e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d16abc56cf8b40bfb93be9fd3eee5c7e 2024-11-14T09:47:50,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c2f4ee078a2249b3b57b25a464bf7a37 2024-11-14T09:47:50,288 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into d16abc56cf8b40bfb93be9fd3eee5c7e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:50,288 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:50,288 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577669663; duration=0sec 2024-11-14T09:47:50,288 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:50,289 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:50,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/495e8a8358be4be8b118551a4755700a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a 2024-11-14T09:47:50,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a, entries=200, sequenceid=174, filesize=14.2 K 2024-11-14T09:47:50,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/75f2d5d80b78443f8351b2bcbf5242b3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3 2024-11-14T09:47:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T09:47:50,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c2f4ee078a2249b3b57b25a464bf7a37 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37 2024-11-14T09:47:50,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T09:47:50,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 91a5f3dba40c0b5503a8f1cccd94111e in 666ms, sequenceid=174, compaction requested=false 2024-11-14T09:47:50,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:50,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T09:47:50,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,422 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:47:50,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/0226397c7ec24cd780c8dcbdd7106e1d is 50, key is test_row_0/A:col10/1731577670416/Put/seqid=0 2024-11-14T09:47:50,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741873_1049 (size=12151) 2024-11-14T09:47:50,456 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/0226397c7ec24cd780c8dcbdd7106e1d 2024-11-14T09:47:50,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577730465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577730466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/54655689106e4ef1a596c8f31e78ff4a is 50, key is test_row_0/B:col10/1731577670416/Put/seqid=0 2024-11-14T09:47:50,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741874_1050 (size=12151) 2024-11-14T09:47:50,486 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/54655689106e4ef1a596c8f31e78ff4a 2024-11-14T09:47:50,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d6fc0dbfe33640078ac396db34aaaf46 is 50, key is test_row_0/C:col10/1731577670416/Put/seqid=0 2024-11-14T09:47:50,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741875_1051 (size=12151) 2024-11-14T09:47:50,546 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d6fc0dbfe33640078ac396db34aaaf46 2024-11-14T09:47:50,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/0226397c7ec24cd780c8dcbdd7106e1d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d 2024-11-14T09:47:50,576 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T09:47:50,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/54655689106e4ef1a596c8f31e78ff4a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a 2024-11-14T09:47:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577730575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577730575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,586 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T09:47:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d6fc0dbfe33640078ac396db34aaaf46 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46 2024-11-14T09:47:50,596 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T09:47:50,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577730596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,599 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 91a5f3dba40c0b5503a8f1cccd94111e in 178ms, sequenceid=197, compaction requested=true 2024-11-14T09:47:50,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T09:47:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T09:47:50,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T09:47:50,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0670 sec 2024-11-14T09:47:50,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0800 sec 2024-11-14T09:47:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:50,619 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:50,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T09:47:50,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/61abcf73290a4c2082aa17f47c46ce99 is 50, key is test_row_0/A:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:50,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741876_1052 (size=14541) 2024-11-14T09:47:50,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/61abcf73290a4c2082aa17f47c46ce99 2024-11-14T09:47:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T09:47:50,676 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:50,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T09:47:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:50,679 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:50,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:50,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:50,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4460414ec5944f7982d8d4f17462395e is 50, key is test_row_0/B:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:50,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741877_1053 (size=12151) 2024-11-14T09:47:50,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4460414ec5944f7982d8d4f17462395e 2024-11-14T09:47:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577730721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/32632daec7814774a702a77165b0b7f8 is 50, key is test_row_0/C:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:50,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741878_1054 (size=12151) 2024-11-14T09:47:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577730785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577730785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:47:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577730835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:50,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:47:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:50,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577731045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577731095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577731095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:47:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:51,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:51,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/32632daec7814774a702a77165b0b7f8 2024-11-14T09:47:51,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/61abcf73290a4c2082aa17f47c46ce99 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99 2024-11-14T09:47:51,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99, entries=200, sequenceid=214, filesize=14.2 K 2024-11-14T09:47:51,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4460414ec5944f7982d8d4f17462395e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e 2024-11-14T09:47:51,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-14T09:47:51,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/32632daec7814774a702a77165b0b7f8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8 2024-11-14T09:47:51,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-14T09:47:51,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 91a5f3dba40c0b5503a8f1cccd94111e in 602ms, sequenceid=214, compaction requested=true 2024-11-14T09:47:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:51,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:51,222 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:51,222 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:51,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:51,224 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:51,224 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:51,224 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,224 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09fd49baf8ba40a290cdcd4c3483db17, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=47.8 K 2024-11-14T09:47:51,225 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:51,225 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:51,225 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,225 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cb1034fc6a5443f19a79b24160ef7d8f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=52.5 K 2024-11-14T09:47:51,226 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1034fc6a5443f19a79b24160ef7d8f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:51,227 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fd49baf8ba40a290cdcd4c3483db17, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:51,229 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 495e8a8358be4be8b118551a4755700a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731577669437 2024-11-14T09:47:51,229 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 75f2d5d80b78443f8351b2bcbf5242b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731577669437 2024-11-14T09:47:51,230 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0226397c7ec24cd780c8dcbdd7106e1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731577669763 2024-11-14T09:47:51,231 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 54655689106e4ef1a596c8f31e78ff4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731577669763 2024-11-14T09:47:51,232 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4460414ec5944f7982d8d4f17462395e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:51,232 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61abcf73290a4c2082aa17f47c46ce99, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:51,266 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:51,267 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a is 50, key is test_row_0/A:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:51,277 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:51,278 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4a18be28e5394e648177688fda1890e5 is 50, key is test_row_0/B:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741880_1056 (size=12629) 2024-11-14T09:47:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741879_1055 (size=12629) 2024-11-14T09:47:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:51,315 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4a18be28e5394e648177688fda1890e5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4a18be28e5394e648177688fda1890e5 2024-11-14T09:47:51,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T09:47:51,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,318 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:47:51,322 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a 2024-11-14T09:47:51,331 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 4a18be28e5394e648177688fda1890e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:51,331 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:51,332 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=12, startTime=1731577671222; duration=0sec 2024-11-14T09:47:51,332 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:51,332 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:51,332 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:47:51,335 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into fbfa3c4eadfe4685a7ec1a1c8e390b7a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:51,335 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:51,335 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=12, startTime=1731577671221; duration=0sec 2024-11-14T09:47:51,336 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:47:51,336 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:51,336 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:51,336 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:51,336 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/665854445ad443bc8dd206995fa37552 is 50, key is test_row_0/A:col10/1731577670703/Put/seqid=0 2024-11-14T09:47:51,342 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d16abc56cf8b40bfb93be9fd3eee5c7e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=47.8 K 2024-11-14T09:47:51,343 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d16abc56cf8b40bfb93be9fd3eee5c7e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577668729 2024-11-14T09:47:51,344 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f4ee078a2249b3b57b25a464bf7a37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731577669437 2024-11-14T09:47:51,345 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d6fc0dbfe33640078ac396db34aaaf46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731577669763 2024-11-14T09:47:51,346 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 32632daec7814774a702a77165b0b7f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:51,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:51,379 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#45 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:51,382 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/261864a2d0f04485a651d0a17c42be00 is 50, key is test_row_0/C:col10/1731577670456/Put/seqid=0 2024-11-14T09:47:51,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741881_1057 (size=12151) 2024-11-14T09:47:51,386 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/665854445ad443bc8dd206995fa37552 2024-11-14T09:47:51,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/912023d02d34494783231ead48d2fb05 is 50, key is test_row_0/B:col10/1731577670703/Put/seqid=0 2024-11-14T09:47:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741882_1058 (size=12629) 2024-11-14T09:47:51,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577731453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741883_1059 (size=12151) 2024-11-14T09:47:51,480 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/912023d02d34494783231ead48d2fb05 2024-11-14T09:47:51,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b567b3d205e6427ebfeca197e562d53c is 50, key is test_row_0/C:col10/1731577670703/Put/seqid=0 2024-11-14T09:47:51,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741884_1060 (size=12151) 2024-11-14T09:47:51,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577731565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577731605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577731605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577731614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,618 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577731775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:51,869 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/261864a2d0f04485a651d0a17c42be00 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/261864a2d0f04485a651d0a17c42be00 2024-11-14T09:47:51,888 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 261864a2d0f04485a651d0a17c42be00(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:51,889 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:51,889 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=12, startTime=1731577671222; duration=0sec 2024-11-14T09:47:51,889 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:51,890 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577731916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:51,921 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:51,925 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b567b3d205e6427ebfeca197e562d53c 2024-11-14T09:47:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/665854445ad443bc8dd206995fa37552 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552 2024-11-14T09:47:51,941 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:47:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/912023d02d34494783231ead48d2fb05 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05 2024-11-14T09:47:51,953 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:47:51,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b567b3d205e6427ebfeca197e562d53c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c 2024-11-14T09:47:51,970 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:47:51,971 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 91a5f3dba40c0b5503a8f1cccd94111e in 653ms, sequenceid=233, compaction requested=false 2024-11-14T09:47:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T09:47:51,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T09:47:51,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T09:47:51,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2930 sec 2024-11-14T09:47:51,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3010 sec 2024-11-14T09:47:52,088 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:52,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T09:47:52,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/93c9f8299bb34c2c88d9e299840d0f57 is 50, key is test_row_0/A:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741885_1061 (size=14541) 2024-11-14T09:47:52,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/93c9f8299bb34c2c88d9e299840d0f57 2024-11-14T09:47:52,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/358e0f86485343e7a36ec5096b5dd6c0 is 50, key is test_row_0/B:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741886_1062 (size=12151) 2024-11-14T09:47:52,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/358e0f86485343e7a36ec5096b5dd6c0 2024-11-14T09:47:52,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/36d89462c24b48bcb4f0af8092fcd1c6 is 50, key is test_row_0/C:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577732196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741887_1063 (size=12151) 2024-11-14T09:47:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577732305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577732515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/36d89462c24b48bcb4f0af8092fcd1c6 2024-11-14T09:47:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577732615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577732615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,617 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:52,617 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/93c9f8299bb34c2c88d9e299840d0f57 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57 2024-11-14T09:47:52,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57, entries=200, sequenceid=254, filesize=14.2 K 2024-11-14T09:47:52,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/358e0f86485343e7a36ec5096b5dd6c0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0 2024-11-14T09:47:52,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T09:47:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/36d89462c24b48bcb4f0af8092fcd1c6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6 2024-11-14T09:47:52,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-14T09:47:52,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 91a5f3dba40c0b5503a8f1cccd94111e in 563ms, sequenceid=254, compaction requested=true 2024-11-14T09:47:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:52,652 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:52,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:52,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:47:52,653 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:52,653 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:52,654 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:52,654 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:52,654 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=38.4 K 2024-11-14T09:47:52,655 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbfa3c4eadfe4685a7ec1a1c8e390b7a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:52,655 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665854445ad443bc8dd206995fa37552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577670670 2024-11-14T09:47:52,655 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:52,655 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:52,655 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:52,656 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4a18be28e5394e648177688fda1890e5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.1 K 2024-11-14T09:47:52,656 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c9f8299bb34c2c88d9e299840d0f57, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:52,656 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a18be28e5394e648177688fda1890e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:52,657 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 912023d02d34494783231ead48d2fb05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577670670 2024-11-14T09:47:52,657 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 358e0f86485343e7a36ec5096b5dd6c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:52,670 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:52,670 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:52,671 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a8884f3107e441d4902f52c8ac6e481a is 50, key is test_row_0/A:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,671 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/bdb7c3610f534d0c9172b221197aa036 is 50, key is test_row_0/B:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741888_1064 (size=12731) 2024-11-14T09:47:52,704 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/bdb7c3610f534d0c9172b221197aa036 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bdb7c3610f534d0c9172b221197aa036 2024-11-14T09:47:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741889_1065 (size=12731) 2024-11-14T09:47:52,739 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into bdb7c3610f534d0c9172b221197aa036(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:52,739 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:52,739 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577672652; duration=0sec 2024-11-14T09:47:52,739 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:52,739 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:52,739 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:52,742 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:52,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:52,743 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:52,743 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/261864a2d0f04485a651d0a17c42be00, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.1 K 2024-11-14T09:47:52,747 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 261864a2d0f04485a651d0a17c42be00, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731577670456 2024-11-14T09:47:52,748 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b567b3d205e6427ebfeca197e562d53c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577670670 2024-11-14T09:47:52,751 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a8884f3107e441d4902f52c8ac6e481a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a8884f3107e441d4902f52c8ac6e481a 2024-11-14T09:47:52,750 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 36d89462c24b48bcb4f0af8092fcd1c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:52,761 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into a8884f3107e441d4902f52c8ac6e481a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:52,761 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:52,761 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577672652; duration=0sec 2024-11-14T09:47:52,761 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:52,761 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:52,765 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:52,766 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/0f5d229cffe049d3b317f3237202c230 is 50, key is test_row_0/C:col10/1731577671448/Put/seqid=0 2024-11-14T09:47:52,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741890_1066 (size=12731) 2024-11-14T09:47:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T09:47:52,826 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:52,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T09:47:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:52,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:47:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:52,830 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:52,833 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/0f5d229cffe049d3b317f3237202c230 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0f5d229cffe049d3b317f3237202c230 2024-11-14T09:47:52,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:52,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:52,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b22918d7cede49e8a53fec30a6a3e4f6 is 50, key is test_row_0/A:col10/1731577672826/Put/seqid=0 2024-11-14T09:47:52,850 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 0f5d229cffe049d3b317f3237202c230(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:52,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:52,851 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577672653; duration=0sec 2024-11-14T09:47:52,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:52,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:52,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741891_1067 (size=14741) 2024-11-14T09:47:52,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b22918d7cede49e8a53fec30a6a3e4f6 2024-11-14T09:47:52,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/dd45fc1abdff4754bd8d84acf3ec0900 is 50, key is test_row_0/B:col10/1731577672826/Put/seqid=0 2024-11-14T09:47:52,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577732919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:52,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741892_1068 (size=12301) 2024-11-14T09:47:52,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/dd45fc1abdff4754bd8d84acf3ec0900 2024-11-14T09:47:52,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/688b699a9de44581a60a19673f17fe9a is 50, key is test_row_0/C:col10/1731577672826/Put/seqid=0 2024-11-14T09:47:52,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:52,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741893_1069 (size=12301) 2024-11-14T09:47:53,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577733035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577733245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/688b699a9de44581a60a19673f17fe9a 2024-11-14T09:47:53,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b22918d7cede49e8a53fec30a6a3e4f6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6 2024-11-14T09:47:53,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6, entries=200, sequenceid=275, filesize=14.4 K 2024-11-14T09:47:53,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/dd45fc1abdff4754bd8d84acf3ec0900 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900 2024-11-14T09:47:53,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:47:53,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/688b699a9de44581a60a19673f17fe9a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a 2024-11-14T09:47:53,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:53,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T09:47:53,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 91a5f3dba40c0b5503a8f1cccd94111e in 636ms, sequenceid=275, compaction requested=false 2024-11-14T09:47:53,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:53,556 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:53,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T09:47:53,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/387cea94bcd1499bbbb728ed7fb90406 is 50, key is test_row_0/A:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:53,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741894_1070 (size=14741) 2024-11-14T09:47:53,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/387cea94bcd1499bbbb728ed7fb90406 2024-11-14T09:47:53,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/8c4591e8e670473da1262e7efd3274f9 is 50, key is test_row_0/B:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:53,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741895_1071 (size=12301) 2024-11-14T09:47:53,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/8c4591e8e670473da1262e7efd3274f9 2024-11-14T09:47:53,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/cec395c084a74c34a72166d94f32d1c8 is 50, key is test_row_0/C:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741896_1072 (size=12301) 2024-11-14T09:47:53,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577733634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577733635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,639 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577733744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:53,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:53,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:53,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577733954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:54,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/cec395c084a74c34a72166d94f32d1c8 2024-11-14T09:47:54,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/387cea94bcd1499bbbb728ed7fb90406 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406 2024-11-14T09:47:54,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406, entries=200, sequenceid=294, filesize=14.4 K 2024-11-14T09:47:54,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/8c4591e8e670473da1262e7efd3274f9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9 2024-11-14T09:47:54,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T09:47:54,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/cec395c084a74c34a72166d94f32d1c8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8 2024-11-14T09:47:54,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:54,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:54,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8, entries=150, sequenceid=294, filesize=12.0 K 2024-11-14T09:47:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:54,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 91a5f3dba40c0b5503a8f1cccd94111e in 515ms, sequenceid=294, compaction requested=true 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:54,071 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:54,071 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:54,072 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:54,072 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:54,073 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:54,073 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:54,073 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,073 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,073 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bdb7c3610f534d0c9172b221197aa036, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.5 K 2024-11-14T09:47:54,073 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a8884f3107e441d4902f52c8ac6e481a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=41.2 K 2024-11-14T09:47:54,073 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8884f3107e441d4902f52c8ac6e481a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:54,073 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb7c3610f534d0c9172b221197aa036, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:54,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b22918d7cede49e8a53fec30a6a3e4f6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731577672162 2024-11-14T09:47:54,074 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting dd45fc1abdff4754bd8d84acf3ec0900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731577672162 2024-11-14T09:47:54,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387cea94bcd1499bbbb728ed7fb90406, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672889 2024-11-14T09:47:54,074 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4591e8e670473da1262e7efd3274f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672901 2024-11-14T09:47:54,082 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:54,082 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:54,082 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3c9e29ff5d7f43b88fba8dca45ce8153 is 50, key is test_row_0/B:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:54,083 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/e15cc4efb47141a18ea739633d58021a is 50, key is test_row_0/A:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:54,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741897_1073 (size=12983) 2024-11-14T09:47:54,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741898_1074 (size=12983) 2024-11-14T09:47:54,104 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3c9e29ff5d7f43b88fba8dca45ce8153 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3c9e29ff5d7f43b88fba8dca45ce8153 2024-11-14T09:47:54,111 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 3c9e29ff5d7f43b88fba8dca45ce8153(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:54,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:54,111 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577674071; duration=0sec 2024-11-14T09:47:54,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:54,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:54,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:54,113 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:54,113 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:54,113 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,114 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0f5d229cffe049d3b317f3237202c230, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.5 K 2024-11-14T09:47:54,114 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5d229cffe049d3b317f3237202c230, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731577671413 2024-11-14T09:47:54,114 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 688b699a9de44581a60a19673f17fe9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731577672162 2024-11-14T09:47:54,115 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting cec395c084a74c34a72166d94f32d1c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672901 2024-11-14T09:47:54,130 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:54,130 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/042aeeeddcde45fca1a8d249f795e48b is 50, key is test_row_0/C:col10/1731577673555/Put/seqid=0 2024-11-14T09:47:54,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741899_1075 (size=12983) 2024-11-14T09:47:54,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T09:47:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,223 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T09:47:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/00874c4a3b5e4d578d91cbecc649d0e3 is 50, key is test_row_0/A:col10/1731577673633/Put/seqid=0 2024-11-14T09:47:54,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741900_1076 (size=12301) 2024-11-14T09:47:54,233 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/00874c4a3b5e4d578d91cbecc649d0e3 2024-11-14T09:47:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02c573f7538e46828c3a53d96635cca9 is 50, key is test_row_0/B:col10/1731577673633/Put/seqid=0 2024-11-14T09:47:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741901_1077 (size=12301) 2024-11-14T09:47:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:54,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:54,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577734327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:54,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577734434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:54,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:54,502 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/e15cc4efb47141a18ea739633d58021a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e15cc4efb47141a18ea739633d58021a 2024-11-14T09:47:54,508 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into e15cc4efb47141a18ea739633d58021a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:54,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:54,508 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577674071; duration=0sec 2024-11-14T09:47:54,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:54,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:54,544 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/042aeeeddcde45fca1a8d249f795e48b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/042aeeeddcde45fca1a8d249f795e48b 2024-11-14T09:47:54,558 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 042aeeeddcde45fca1a8d249f795e48b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:54,558 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:54,558 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577674071; duration=0sec 2024-11-14T09:47:54,558 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:54,558 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577734635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577734635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:54,636 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:54,637 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577734644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:54,653 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02c573f7538e46828c3a53d96635cca9 2024-11-14T09:47:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/37bdca6ca2b74bbd8d31955c19672789 is 50, key is test_row_0/C:col10/1731577673633/Put/seqid=0 2024-11-14T09:47:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741902_1078 (size=12301) 2024-11-14T09:47:54,680 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/37bdca6ca2b74bbd8d31955c19672789 2024-11-14T09:47:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/00874c4a3b5e4d578d91cbecc649d0e3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3 2024-11-14T09:47:54,704 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T09:47:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02c573f7538e46828c3a53d96635cca9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9 2024-11-14T09:47:54,716 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T09:47:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/37bdca6ca2b74bbd8d31955c19672789 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789 2024-11-14T09:47:54,726 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789, entries=150, sequenceid=313, filesize=12.0 K 2024-11-14T09:47:54,727 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 91a5f3dba40c0b5503a8f1cccd94111e in 504ms, sequenceid=313, compaction requested=false 2024-11-14T09:47:54,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:54,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:54,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T09:47:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T09:47:54,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T09:47:54,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8970 sec 2024-11-14T09:47:54,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.9110 sec 2024-11-14T09:47:54,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:54,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:47:54,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/79ec7ef792824d5699bcc19979583bae is 50, key is test_row_0/A:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741903_1079 (size=12301) 2024-11-14T09:47:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T09:47:54,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:54,980 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T09:47:54,984 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:54,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:54,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:55,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577735033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:55,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:47:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577735150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:55,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:47:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577735365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:55,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/79ec7ef792824d5699bcc19979583bae 2024-11-14T09:47:55,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/e9ea266991234eb5b9e214467f68d388 is 50, key is test_row_0/B:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:55,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741904_1080 (size=12301) 2024-11-14T09:47:55,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/e9ea266991234eb5b9e214467f68d388 2024-11-14T09:47:55,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/e3fb7d5952db4d34a46b70500da59861 is 50, key is test_row_0/C:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741905_1081 (size=12301) 2024-11-14T09:47:55,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/e3fb7d5952db4d34a46b70500da59861 2024-11-14T09:47:55,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/79ec7ef792824d5699bcc19979583bae as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae 2024-11-14T09:47:55,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae, entries=150, sequenceid=334, filesize=12.0 K 2024-11-14T09:47:55,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/e9ea266991234eb5b9e214467f68d388 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388 2024-11-14T09:47:55,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388, entries=150, sequenceid=334, filesize=12.0 K 2024-11-14T09:47:55,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/e3fb7d5952db4d34a46b70500da59861 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861 2024-11-14T09:47:55,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:47:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:55,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861, entries=150, sequenceid=334, filesize=12.0 K 2024-11-14T09:47:55,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 91a5f3dba40c0b5503a8f1cccd94111e in 505ms, sequenceid=334, compaction requested=true 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:55,464 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:55,464 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:55,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:55,465 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:55,466 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:55,466 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,466 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e15cc4efb47141a18ea739633d58021a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.7 K 2024-11-14T09:47:55,466 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:55,466 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:55,466 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,467 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3c9e29ff5d7f43b88fba8dca45ce8153, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.7 K 2024-11-14T09:47:55,468 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9e29ff5d7f43b88fba8dca45ce8153, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672901 2024-11-14T09:47:55,468 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15cc4efb47141a18ea739633d58021a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672901 2024-11-14T09:47:55,469 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c573f7538e46828c3a53d96635cca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731577673606 2024-11-14T09:47:55,469 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00874c4a3b5e4d578d91cbecc649d0e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731577673606 2024-11-14T09:47:55,470 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e9ea266991234eb5b9e214467f68d388, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:55,470 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ec7ef792824d5699bcc19979583bae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:55,490 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:55,491 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/22b99e6bef6a47deb67b9e9835f29c84 is 50, key is test_row_0/B:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:55,498 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:55,499 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ddcb768ecf564d4e95fcae1a6a4879d1 is 50, key is test_row_0/A:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741906_1082 (size=13085) 2024-11-14T09:47:55,532 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/22b99e6bef6a47deb67b9e9835f29c84 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/22b99e6bef6a47deb67b9e9835f29c84 2024-11-14T09:47:55,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741907_1083 (size=13085) 2024-11-14T09:47:55,543 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ddcb768ecf564d4e95fcae1a6a4879d1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ddcb768ecf564d4e95fcae1a6a4879d1 2024-11-14T09:47:55,546 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 22b99e6bef6a47deb67b9e9835f29c84(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:55,550 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:55,550 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577675464; duration=0sec 2024-11-14T09:47:55,551 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:55,551 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:55,551 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:55,555 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into ddcb768ecf564d4e95fcae1a6a4879d1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:55,556 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:55,556 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577675464; duration=0sec 2024-11-14T09:47:55,556 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:55,556 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:55,557 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:55,557 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:55,558 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,558 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/042aeeeddcde45fca1a8d249f795e48b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.7 K 2024-11-14T09:47:55,558 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 042aeeeddcde45fca1a8d249f795e48b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731577672901 2024-11-14T09:47:55,559 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 37bdca6ca2b74bbd8d31955c19672789, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731577673606 2024-11-14T09:47:55,559 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e3fb7d5952db4d34a46b70500da59861, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:55,580 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#71 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:55,581 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/31c7c9ebd59843b3a1670e3ee2969fbe is 50, key is test_row_0/C:col10/1731577674296/Put/seqid=0 2024-11-14T09:47:55,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741908_1084 (size=13085) 2024-11-14T09:47:55,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T09:47:55,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:55,607 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T09:47:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:55,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/9f6b43a7c288405e8ea1359ee7c29912 is 50, key is test_row_0/A:col10/1731577675026/Put/seqid=0 2024-11-14T09:47:55,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741909_1085 (size=12301) 2024-11-14T09:47:55,651 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/9f6b43a7c288405e8ea1359ee7c29912 2024-11-14T09:47:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0abdd7f1210341d3a537592271b5ae00 is 50, key is test_row_0/B:col10/1731577675026/Put/seqid=0 2024-11-14T09:47:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:55,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:55,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741910_1086 (size=12301) 2024-11-14T09:47:55,713 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0abdd7f1210341d3a537592271b5ae00 2024-11-14T09:47:55,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33bb19e04d294e46a217e4a122dd47ad is 50, key is test_row_0/C:col10/1731577675026/Put/seqid=0 2024-11-14T09:47:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577735756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:55,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741911_1087 (size=12301) 2024-11-14T09:47:55,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577735864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55804 deadline: 1731577735935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:55,936 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:56,007 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/31c7c9ebd59843b3a1670e3ee2969fbe as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/31c7c9ebd59843b3a1670e3ee2969fbe 2024-11-14T09:47:56,015 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 31c7c9ebd59843b3a1670e3ee2969fbe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:56,015 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:56,015 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577675464; duration=0sec 2024-11-14T09:47:56,015 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:56,016 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:56,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:56,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577736075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:56,163 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33bb19e04d294e46a217e4a122dd47ad 2024-11-14T09:47:56,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/9f6b43a7c288405e8ea1359ee7c29912 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912 2024-11-14T09:47:56,176 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:47:56,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0abdd7f1210341d3a537592271b5ae00 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00 2024-11-14T09:47:56,186 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:47:56,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33bb19e04d294e46a217e4a122dd47ad as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad 2024-11-14T09:47:56,194 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad, entries=150, sequenceid=353, filesize=12.0 K 2024-11-14T09:47:56,195 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 91a5f3dba40c0b5503a8f1cccd94111e in 588ms, sequenceid=353, compaction requested=false 2024-11-14T09:47:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T09:47:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T09:47:56,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T09:47:56,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2110 sec 2024-11-14T09:47:56,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2260 sec 2024-11-14T09:47:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:56,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:56,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T09:47:56,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ab7546c2829940eea9c6b0824ab14f17 is 50, key is test_row_0/A:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:56,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741912_1088 (size=14741) 2024-11-14T09:47:56,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ab7546c2829940eea9c6b0824ab14f17 2024-11-14T09:47:56,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3e0702c013e2488782c16922cd8ad621 is 50, key is test_row_0/B:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741913_1089 (size=12301) 2024-11-14T09:47:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577736447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577736554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577736765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:56,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3e0702c013e2488782c16922cd8ad621 2024-11-14T09:47:56,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33e011ee7bec4491aa9380d6b3551031 is 50, key is test_row_0/C:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:56,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741914_1090 (size=12301) 2024-11-14T09:47:57,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577737075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T09:47:57,135 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:57,135 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T09:47:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:57,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:57,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:57,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:57,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33e011ee7bec4491aa9380d6b3551031 2024-11-14T09:47:57,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ab7546c2829940eea9c6b0824ab14f17 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17 2024-11-14T09:47:57,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17, entries=200, sequenceid=374, filesize=14.4 K 2024-11-14T09:47:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/3e0702c013e2488782c16922cd8ad621 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621 2024-11-14T09:47:57,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T09:47:57,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/33e011ee7bec4491aa9380d6b3551031 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031 2024-11-14T09:47:57,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:47:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:47:57,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T09:47:57,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 91a5f3dba40c0b5503a8f1cccd94111e in 908ms, sequenceid=374, compaction requested=true 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:57,295 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:57,295 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:57,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:57,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:57,296 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:57,297 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:57,297 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,297 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ddcb768ecf564d4e95fcae1a6a4879d1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.2 K 2024-11-14T09:47:57,297 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:57,297 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:57,297 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,297 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/22b99e6bef6a47deb67b9e9835f29c84, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.8 K 2024-11-14T09:47:57,297 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddcb768ecf564d4e95fcae1a6a4879d1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:57,298 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b99e6bef6a47deb67b9e9835f29c84, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:57,298 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0abdd7f1210341d3a537592271b5ae00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731577675014 2024-11-14T09:47:57,298 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6b43a7c288405e8ea1359ee7c29912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731577675014 2024-11-14T09:47:57,298 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0702c013e2488782c16922cd8ad621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:57,298 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7546c2829940eea9c6b0824ab14f17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:57,310 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:57,311 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/971dbde784e341f2b52f1b5a326197c3 is 50, key is test_row_0/B:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:57,313 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:57,314 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/5d2c9c78849e46f484900cd4b029cac4 is 50, key is test_row_0/A:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:57,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741915_1091 (size=13187) 2024-11-14T09:47:57,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741916_1092 (size=13187) 2024-11-14T09:47:57,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T09:47:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,444 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:47:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b02b9f8cb10d475c9f5d9656d389f83a is 50, key is test_row_0/A:col10/1731577676435/Put/seqid=0 2024-11-14T09:47:57,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741917_1093 (size=12301) 2024-11-14T09:47:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:57,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577737658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:57,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55820 deadline: 1731577737675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:57,677 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:57,741 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/971dbde784e341f2b52f1b5a326197c3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/971dbde784e341f2b52f1b5a326197c3 2024-11-14T09:47:57,745 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/5d2c9c78849e46f484900cd4b029cac4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5d2c9c78849e46f484900cd4b029cac4 2024-11-14T09:47:57,749 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 971dbde784e341f2b52f1b5a326197c3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:57,749 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:57,749 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577677295; duration=0sec 2024-11-14T09:47:57,750 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:57,750 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:57,750 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:57,752 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:57,752 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into 5d2c9c78849e46f484900cd4b029cac4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:57,752 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:57,753 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:57,753 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:57,753 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577677295; duration=0sec 2024-11-14T09:47:57,753 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/31c7c9ebd59843b3a1670e3ee2969fbe, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.8 K 2024-11-14T09:47:57,753 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:57,753 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:57,753 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 31c7c9ebd59843b3a1670e3ee2969fbe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731577674296 2024-11-14T09:47:57,754 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 33bb19e04d294e46a217e4a122dd47ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731577675014 2024-11-14T09:47:57,754 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 33e011ee7bec4491aa9380d6b3551031, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:57,763 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:57,764 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c3d6b7218017485fa8a84d0f28056429 is 50, key is test_row_0/C:col10/1731577676385/Put/seqid=0 2024-11-14T09:47:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:57,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577737765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:57,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741918_1094 (size=13187) 2024-11-14T09:47:57,854 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b02b9f8cb10d475c9f5d9656d389f83a 2024-11-14T09:47:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/588ee6c1c4b94ff79505ce4f7cfb3924 is 50, key is test_row_0/B:col10/1731577676435/Put/seqid=0 2024-11-14T09:47:57,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741919_1095 (size=12301) 2024-11-14T09:47:57,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577737975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:57,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:57,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:58,186 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c3d6b7218017485fa8a84d0f28056429 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c3d6b7218017485fa8a84d0f28056429 2024-11-14T09:47:58,194 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into c3d6b7218017485fa8a84d0f28056429(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:58,194 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:58,194 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577677295; duration=0sec 2024-11-14T09:47:58,194 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:58,194 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:58,269 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/588ee6c1c4b94ff79505ce4f7cfb3924 2024-11-14T09:47:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:58,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577738285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:58,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4eae2a79be194348ab8a0b4522d5681f is 50, key is test_row_0/C:col10/1731577676435/Put/seqid=0 2024-11-14T09:47:58,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741920_1096 (size=12301) 2024-11-14T09:47:58,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55810 deadline: 1731577738645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:58,647 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:58,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55848 deadline: 1731577738675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:58,676 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 91a5f3dba40c0b5503a8f1cccd94111e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:47:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:58,710 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4eae2a79be194348ab8a0b4522d5681f 2024-11-14T09:47:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/b02b9f8cb10d475c9f5d9656d389f83a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a 2024-11-14T09:47:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a, entries=150, sequenceid=390, filesize=12.0 K 2024-11-14T09:47:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/588ee6c1c4b94ff79505ce4f7cfb3924 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924 2024-11-14T09:47:58,737 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924, entries=150, sequenceid=390, filesize=12.0 K 2024-11-14T09:47:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4eae2a79be194348ab8a0b4522d5681f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f 2024-11-14T09:47:58,747 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f, entries=150, sequenceid=390, filesize=12.0 K 2024-11-14T09:47:58,749 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 91a5f3dba40c0b5503a8f1cccd94111e in 1305ms, sequenceid=390, compaction requested=false 2024-11-14T09:47:58,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:58,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:58,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T09:47:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T09:47:58,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T09:47:58,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-14T09:47:58,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6210 sec 2024-11-14T09:47:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:58,796 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:58,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:47:58,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/cd832d6716104608bddbb1ba33b5970a is 50, key is test_row_0/A:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:58,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741921_1097 (size=14741) 2024-11-14T09:47:58,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577738848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:58,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577738955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:59,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577739165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:59,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/cd832d6716104608bddbb1ba33b5970a 2024-11-14T09:47:59,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/70b0761b71ee40cfb48383df6db19b9d is 50, key is test_row_0/B:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:59,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741922_1098 (size=12301) 2024-11-14T09:47:59,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/70b0761b71ee40cfb48383df6db19b9d 2024-11-14T09:47:59,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5247447a11264919bf83d2c861eb2c8c is 50, key is test_row_0/C:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T09:47:59,285 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:59,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T09:47:59,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741923_1099 (size=12301) 2024-11-14T09:47:59,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:59,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5247447a11264919bf83d2c861eb2c8c 2024-11-14T09:47:59,289 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:59,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:47:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:47:59,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/cd832d6716104608bddbb1ba33b5970a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a 2024-11-14T09:47:59,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a, entries=200, sequenceid=414, filesize=14.4 K 2024-11-14T09:47:59,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/70b0761b71ee40cfb48383df6db19b9d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d 2024-11-14T09:47:59,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T09:47:59,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/5247447a11264919bf83d2c861eb2c8c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c 2024-11-14T09:47:59,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c, entries=150, sequenceid=414, filesize=12.0 K 2024-11-14T09:47:59,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 91a5f3dba40c0b5503a8f1cccd94111e in 530ms, sequenceid=414, compaction requested=true 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:59,326 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:59,326 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:47:59,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:59,327 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:59,327 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:47:59,327 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:59,328 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/971dbde784e341f2b52f1b5a326197c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.9 K 2024-11-14T09:47:59,328 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:59,328 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:47:59,328 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:59,328 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5d2c9c78849e46f484900cd4b029cac4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.3 K 2024-11-14T09:47:59,329 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 971dbde784e341f2b52f1b5a326197c3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:59,330 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d2c9c78849e46f484900cd4b029cac4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:59,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 588ee6c1c4b94ff79505ce4f7cfb3924, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731577676435 2024-11-14T09:47:59,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b0761b71ee40cfb48383df6db19b9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:47:59,330 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b02b9f8cb10d475c9f5d9656d389f83a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731577676435 2024-11-14T09:47:59,331 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd832d6716104608bddbb1ba33b5970a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:47:59,346 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:59,347 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a7d6663219b4420c8627ecf7f5e57c40 is 50, key is test_row_0/A:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:59,354 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:59,355 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/f40af7cb414640378029e6c4773e0e10 is 50, key is test_row_0/B:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741924_1100 (size=13289) 2024-11-14T09:47:59,373 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a7d6663219b4420c8627ecf7f5e57c40 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a7d6663219b4420c8627ecf7f5e57c40 2024-11-14T09:47:59,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741925_1101 (size=13289) 2024-11-14T09:47:59,382 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into a7d6663219b4420c8627ecf7f5e57c40(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:59,382 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:59,382 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577679326; duration=0sec 2024-11-14T09:47:59,383 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:47:59,383 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:47:59,383 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:47:59,383 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/f40af7cb414640378029e6c4773e0e10 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/f40af7cb414640378029e6c4773e0e10 2024-11-14T09:47:59,385 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:47:59,385 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:47:59,385 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:59,385 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c3d6b7218017485fa8a84d0f28056429, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=36.9 K 2024-11-14T09:47:59,386 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d6b7218017485fa8a84d0f28056429, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731577675719 2024-11-14T09:47:59,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eae2a79be194348ab8a0b4522d5681f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1731577676435 2024-11-14T09:47:59,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5247447a11264919bf83d2c861eb2c8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:47:59,392 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into f40af7cb414640378029e6c4773e0e10(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:59,392 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:59,392 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577679326; duration=0sec 2024-11-14T09:47:59,392 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:59,393 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:47:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:47:59,398 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#89 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:47:59,399 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/826c807fe770430d9ab13d898de414eb is 50, key is test_row_0/C:col10/1731577677643/Put/seqid=0 2024-11-14T09:47:59,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741926_1102 (size=13289) 2024-11-14T09:47:59,426 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/826c807fe770430d9ab13d898de414eb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/826c807fe770430d9ab13d898de414eb 2024-11-14T09:47:59,435 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 826c807fe770430d9ab13d898de414eb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:47:59,435 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:59,435 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577679326; duration=0sec 2024-11-14T09:47:59,435 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:47:59,435 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:47:59,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T09:47:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:59,443 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:47:59,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/06e4bf8b72384f539a61d221a196f52c is 50, key is test_row_0/A:col10/1731577678835/Put/seqid=0 2024-11-14T09:47:59,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741927_1103 (size=12301) 2024-11-14T09:47:59,466 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/06e4bf8b72384f539a61d221a196f52c 2024-11-14T09:47:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ed4668e0e6ee4153b96fa9902c4f267f is 50, key is test_row_0/B:col10/1731577678835/Put/seqid=0 2024-11-14T09:47:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:59,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:47:59,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741928_1104 (size=12301) 2024-11-14T09:47:59,522 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ed4668e0e6ee4153b96fa9902c4f267f 2024-11-14T09:47:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/6ed19d1307124923972660c88970ee68 is 50, key is test_row_0/C:col10/1731577678835/Put/seqid=0 2024-11-14T09:47:59,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741929_1105 (size=12301) 2024-11-14T09:47:59,557 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/6ed19d1307124923972660c88970ee68 2024-11-14T09:47:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/06e4bf8b72384f539a61d221a196f52c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c 2024-11-14T09:47:59,575 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c, entries=150, sequenceid=432, filesize=12.0 K 2024-11-14T09:47:59,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/ed4668e0e6ee4153b96fa9902c4f267f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f 2024-11-14T09:47:59,586 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f, entries=150, sequenceid=432, filesize=12.0 K 2024-11-14T09:47:59,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/6ed19d1307124923972660c88970ee68 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68 2024-11-14T09:47:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577739587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:59,596 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68, entries=150, sequenceid=432, filesize=12.0 K 2024-11-14T09:47:59,597 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 91a5f3dba40c0b5503a8f1cccd94111e in 154ms, sequenceid=432, compaction requested=false 2024-11-14T09:47:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:47:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:47:59,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T09:47:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T09:47:59,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T09:47:59,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 310 msec 2024-11-14T09:47:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:47:59,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 317 msec 2024-11-14T09:47:59,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:47:59,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:47:59,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:47:59,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ae0c927f3db64f48b7a116c1a5b1defe is 50, key is test_row_0/A:col10/1731577679581/Put/seqid=0 2024-11-14T09:47:59,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741930_1106 (size=14741) 2024-11-14T09:47:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577739745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:47:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577739854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:47:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:47:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T09:47:59,915 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:47:59,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:47:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T09:47:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:47:59,918 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:47:59,918 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:47:59,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:48:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577740064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:00,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:48:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ae0c927f3db64f48b7a116c1a5b1defe 2024-11-14T09:48:00,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02346820f87d4b748d63564c43b3b149 is 50, key is test_row_0/B:col10/1731577679581/Put/seqid=0 2024-11-14T09:48:00,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741931_1107 (size=12301) 2024-11-14T09:48:00,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:48:00,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:00,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:48:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577740375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:00,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:48:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:48:00,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:00,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:00,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02346820f87d4b748d63564c43b3b149 2024-11-14T09:48:00,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/044ec8c635194cd49c00a0ed9ca5b4b2 is 50, key is test_row_0/C:col10/1731577679581/Put/seqid=0 2024-11-14T09:48:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:48:00,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741932_1108 (size=12301) 2024-11-14T09:48:00,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/044ec8c635194cd49c00a0ed9ca5b4b2 2024-11-14T09:48:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ae0c927f3db64f48b7a116c1a5b1defe as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe 2024-11-14T09:48:00,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe, entries=200, sequenceid=454, filesize=14.4 K 2024-11-14T09:48:00,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/02346820f87d4b748d63564c43b3b149 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149 2024-11-14T09:48:00,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149, entries=150, sequenceid=454, filesize=12.0 K 2024-11-14T09:48:00,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/044ec8c635194cd49c00a0ed9ca5b4b2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2 2024-11-14T09:48:00,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2, entries=150, sequenceid=454, filesize=12.0 K 2024-11-14T09:48:00,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 91a5f3dba40c0b5503a8f1cccd94111e in 888ms, sequenceid=454, compaction requested=true 2024-11-14T09:48:00,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:00,587 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:00,587 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:00,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:00,588 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:00,588 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:00,588 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:48:00,588 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:48:00,588 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,588 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,589 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a7d6663219b4420c8627ecf7f5e57c40, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.4 K 2024-11-14T09:48:00,589 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/f40af7cb414640378029e6c4773e0e10, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.0 K 2024-11-14T09:48:00,589 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d6663219b4420c8627ecf7f5e57c40, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:48:00,589 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f40af7cb414640378029e6c4773e0e10, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:48:00,589 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e4bf8b72384f539a61d221a196f52c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731577678835 2024-11-14T09:48:00,590 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ed4668e0e6ee4153b96fa9902c4f267f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731577678835 2024-11-14T09:48:00,590 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae0c927f3db64f48b7a116c1a5b1defe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679572 2024-11-14T09:48:00,590 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 02346820f87d4b748d63564c43b3b149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679581 2024-11-14T09:48:00,600 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:00,601 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/b6724191aab24f72b3df778357ae2181 is 50, key is test_row_0/B:col10/1731577679581/Put/seqid=0 2024-11-14T09:48:00,606 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:00,607 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/8b5678bcd6664c7f82234376af6da5b6 is 50, key is test_row_0/A:col10/1731577679581/Put/seqid=0 2024-11-14T09:48:00,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741933_1109 (size=13391) 2024-11-14T09:48:00,621 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/b6724191aab24f72b3df778357ae2181 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/b6724191aab24f72b3df778357ae2181 2024-11-14T09:48:00,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741934_1110 (size=13391) 2024-11-14T09:48:00,634 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/8b5678bcd6664c7f82234376af6da5b6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/8b5678bcd6664c7f82234376af6da5b6 2024-11-14T09:48:00,637 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into b6724191aab24f72b3df778357ae2181(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:00,637 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:00,637 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577680587; duration=0sec 2024-11-14T09:48:00,637 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:00,637 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:48:00,637 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:00,640 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:00,641 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:48:00,641 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,641 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/826c807fe770430d9ab13d898de414eb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.0 K 2024-11-14T09:48:00,641 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 826c807fe770430d9ab13d898de414eb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731577677643 2024-11-14T09:48:00,642 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed19d1307124923972660c88970ee68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731577678835 2024-11-14T09:48:00,643 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 044ec8c635194cd49c00a0ed9ca5b4b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679581 2024-11-14T09:48:00,645 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into 8b5678bcd6664c7f82234376af6da5b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:00,645 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:00,645 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577680586; duration=0sec 2024-11-14T09:48:00,646 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:00,646 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:48:00,663 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:00,664 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/75403b6124594d6cba0f48f3549e3d33 is 50, key is test_row_0/C:col10/1731577679581/Put/seqid=0 2024-11-14T09:48:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741935_1111 (size=13391) 2024-11-14T09:48:00,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T09:48:00,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:00,687 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:48:00,707 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/75403b6124594d6cba0f48f3549e3d33 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/75403b6124594d6cba0f48f3549e3d33 2024-11-14T09:48:00,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/1d84aede37b146e5801e9cd421460510 is 50, key is test_row_0/A:col10/1731577679713/Put/seqid=0 2024-11-14T09:48:00,715 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 75403b6124594d6cba0f48f3549e3d33(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:00,716 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:00,716 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577680587; duration=0sec 2024-11-14T09:48:00,716 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:00,716 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:48:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741936_1112 (size=12301) 2024-11-14T09:48:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:00,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:00,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577740986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:48:01,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577741095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:01,133 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/1d84aede37b146e5801e9cd421460510 2024-11-14T09:48:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/46424564a81c4d15ae5e4f5bcba3b540 is 50, key is test_row_0/B:col10/1731577679713/Put/seqid=0 2024-11-14T09:48:01,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741937_1113 (size=12301) 2024-11-14T09:48:01,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:01,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577741304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:01,571 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/46424564a81c4d15ae5e4f5bcba3b540 2024-11-14T09:48:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1532ea97e00b4803a5fcca05ea555937 is 50, key is test_row_0/C:col10/1731577679713/Put/seqid=0 2024-11-14T09:48:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741938_1114 (size=12301) 2024-11-14T09:48:01,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577741615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:01,992 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1532ea97e00b4803a5fcca05ea555937 2024-11-14T09:48:02,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/1d84aede37b146e5801e9cd421460510 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510 2024-11-14T09:48:02,006 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510, entries=150, sequenceid=470, filesize=12.0 K 2024-11-14T09:48:02,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/46424564a81c4d15ae5e4f5bcba3b540 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540 2024-11-14T09:48:02,015 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540, entries=150, sequenceid=470, filesize=12.0 K 2024-11-14T09:48:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1532ea97e00b4803a5fcca05ea555937 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937 2024-11-14T09:48:02,025 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937, entries=150, sequenceid=470, filesize=12.0 K 2024-11-14T09:48:02,027 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 91a5f3dba40c0b5503a8f1cccd94111e in 1339ms, sequenceid=470, compaction requested=false 2024-11-14T09:48:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T09:48:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T09:48:02,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T09:48:02,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1090 sec 2024-11-14T09:48:02,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1150 sec 2024-11-14T09:48:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T09:48:02,075 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:02,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-14T09:48:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:02,077 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:02,078 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:02,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:02,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:02,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T09:48:02,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ebbba27cd5e2465f9184178da039c4ab is 50, key is test_row_0/A:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:02,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741939_1115 (size=14741) 2024-11-14T09:48:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577742173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:02,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577742285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:02,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577742494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:02,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ebbba27cd5e2465f9184178da039c4ab 2024-11-14T09:48:02,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0fb1dbf09c254da7bbf34045f0120c49 is 50, key is test_row_0/B:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:02,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741940_1116 (size=12301) 2024-11-14T09:48:02,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:02,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577742805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:02,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:02,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:03,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0fb1dbf09c254da7bbf34045f0120c49 2024-11-14T09:48:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1b9036a0fd064192b760634ae734959c is 50, key is test_row_0/C:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:03,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741941_1117 (size=12301) 2024-11-14T09:48:03,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1b9036a0fd064192b760634ae734959c 2024-11-14T09:48:03,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ebbba27cd5e2465f9184178da039c4ab as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab 2024-11-14T09:48:03,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab, entries=200, sequenceid=494, filesize=14.4 K 2024-11-14T09:48:03,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/0fb1dbf09c254da7bbf34045f0120c49 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49 2024-11-14T09:48:03,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T09:48:03,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/1b9036a0fd064192b760634ae734959c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c 2024-11-14T09:48:03,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T09:48:03,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 91a5f3dba40c0b5503a8f1cccd94111e in 934ms, sequenceid=494, compaction requested=true 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:03,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:03,074 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:03,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:48:03,075 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:03,075 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:03,075 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/8b5678bcd6664c7f82234376af6da5b6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.5 K 2024-11-14T09:48:03,075 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/b6724191aab24f72b3df778357ae2181, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.1 K 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5678bcd6664c7f82234376af6da5b6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679581 2024-11-14T09:48:03,075 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b6724191aab24f72b3df778357ae2181, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679581 2024-11-14T09:48:03,076 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d84aede37b146e5801e9cd421460510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731577679713 2024-11-14T09:48:03,076 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 46424564a81c4d15ae5e4f5bcba3b540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731577679713 2024-11-14T09:48:03,076 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebbba27cd5e2465f9184178da039c4ab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:03,076 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fb1dbf09c254da7bbf34045f0120c49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:03,084 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:03,084 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c406eb14c661411691f454a1e9a33b9c is 50, key is test_row_0/A:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:03,086 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:03,087 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/fe4345c726ef47f493d3d3def332c684 is 50, key is test_row_0/B:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:03,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741942_1118 (size=13493) 2024-11-14T09:48:03,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741943_1119 (size=13493) 2024-11-14T09:48:03,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-14T09:48:03,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:03,150 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:48:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/45b78ada0fc545009bbdfce82936efc4 is 50, key is test_row_0/A:col10/1731577682161/Put/seqid=0 2024-11-14T09:48:03,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741944_1120 (size=12301) 2024-11-14T09:48:03,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:03,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. as already flushing 2024-11-14T09:48:03,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:03,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577743387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:03,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577743495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:03,502 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c406eb14c661411691f454a1e9a33b9c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c406eb14c661411691f454a1e9a33b9c 2024-11-14T09:48:03,506 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/fe4345c726ef47f493d3d3def332c684 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/fe4345c726ef47f493d3d3def332c684 2024-11-14T09:48:03,508 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into c406eb14c661411691f454a1e9a33b9c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:03,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:03,509 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577683074; duration=0sec 2024-11-14T09:48:03,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:03,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:48:03,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:03,511 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:03,513 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:48:03,513 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:03,514 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into fe4345c726ef47f493d3d3def332c684(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:03,514 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/75403b6124594d6cba0f48f3549e3d33, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.1 K 2024-11-14T09:48:03,514 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:03,514 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577683074; duration=0sec 2024-11-14T09:48:03,514 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:03,514 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:48:03,514 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75403b6124594d6cba0f48f3549e3d33, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731577679581 2024-11-14T09:48:03,515 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1532ea97e00b4803a5fcca05ea555937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731577679713 2024-11-14T09:48:03,515 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b9036a0fd064192b760634ae734959c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:03,524 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:03,525 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/14dd477313014d3ab8fd7da366a529ba is 50, key is test_row_0/C:col10/1731577680984/Put/seqid=0 2024-11-14T09:48:03,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741945_1121 (size=13493) 2024-11-14T09:48:03,559 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/45b78ada0fc545009bbdfce82936efc4 2024-11-14T09:48:03,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4c7fde202fd1429fb8faa1e3c777f498 is 50, key is test_row_0/B:col10/1731577682161/Put/seqid=0 2024-11-14T09:48:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741946_1122 (size=12301) 2024-11-14T09:48:03,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577743704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:03,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:03,937 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/14dd477313014d3ab8fd7da366a529ba as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/14dd477313014d3ab8fd7da366a529ba 2024-11-14T09:48:03,943 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 14dd477313014d3ab8fd7da366a529ba(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:03,943 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:03,943 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577683074; duration=0sec 2024-11-14T09:48:03,943 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:03,943 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:48:03,971 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4c7fde202fd1429fb8faa1e3c777f498 2024-11-14T09:48:03,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b47388b7268a4ad4a616411afdb2396c is 50, key is test_row_0/C:col10/1731577682161/Put/seqid=0 2024-11-14T09:48:03,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741947_1123 (size=12301) 2024-11-14T09:48:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577744014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:04,384 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b47388b7268a4ad4a616411afdb2396c 2024-11-14T09:48:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/45b78ada0fc545009bbdfce82936efc4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4 2024-11-14T09:48:04,400 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4, entries=150, sequenceid=507, filesize=12.0 K 2024-11-14T09:48:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/4c7fde202fd1429fb8faa1e3c777f498 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498 2024-11-14T09:48:04,419 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498, entries=150, sequenceid=507, filesize=12.0 K 2024-11-14T09:48:04,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/b47388b7268a4ad4a616411afdb2396c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c 2024-11-14T09:48:04,428 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c, entries=150, sequenceid=507, filesize=12.0 K 2024-11-14T09:48:04,429 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 91a5f3dba40c0b5503a8f1cccd94111e in 1279ms, sequenceid=507, compaction requested=false 2024-11-14T09:48:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:04,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:04,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-14T09:48:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-14T09:48:04,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T09:48:04,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3570 sec 2024-11-14T09:48:04,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3630 sec 2024-11-14T09:48:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:04,528 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:04,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:48:04,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ed814234c1034d67a6f76fdf20d48b20 is 50, key is test_row_0/A:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 367 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577744574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:04,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741948_1124 (size=14741) 2024-11-14T09:48:04,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ed814234c1034d67a6f76fdf20d48b20 2024-11-14T09:48:04,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c7709b237db6459094f81714192d01e4 is 50, key is test_row_0/B:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741949_1125 (size=12301) 2024-11-14T09:48:04,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c7709b237db6459094f81714192d01e4 2024-11-14T09:48:04,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/024282fc80104ad4ad9b70da03c9fed6 is 50, key is test_row_0/C:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741950_1126 (size=12301) 2024-11-14T09:48:04,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/024282fc80104ad4ad9b70da03c9fed6 2024-11-14T09:48:04,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/ed814234c1034d67a6f76fdf20d48b20 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20 2024-11-14T09:48:04,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20, entries=200, sequenceid=534, filesize=14.4 K 2024-11-14T09:48:04,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/c7709b237db6459094f81714192d01e4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4 2024-11-14T09:48:04,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4, entries=150, sequenceid=534, filesize=12.0 K 2024-11-14T09:48:04,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/024282fc80104ad4ad9b70da03c9fed6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6 2024-11-14T09:48:04,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6, entries=150, sequenceid=534, filesize=12.0 K 2024-11-14T09:48:04,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 91a5f3dba40c0b5503a8f1cccd94111e in 152ms, sequenceid=534, compaction requested=true 2024-11-14T09:48:04,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:04,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:04,680 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:04,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:04,680 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:04,682 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:04,682 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:48:04,682 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:04,682 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/fe4345c726ef47f493d3d3def332c684, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.2 K 2024-11-14T09:48:04,682 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:04,682 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:48:04,682 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:04,683 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c406eb14c661411691f454a1e9a33b9c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.6 K 2024-11-14T09:48:04,683 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4345c726ef47f493d3d3def332c684, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:04,684 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c7fde202fd1429fb8faa1e3c777f498, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731577682161 2024-11-14T09:48:04,685 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c406eb14c661411691f454a1e9a33b9c, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:04,685 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c7709b237db6459094f81714192d01e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:04,685 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45b78ada0fc545009bbdfce82936efc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731577682161 2024-11-14T09:48:04,686 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed814234c1034d67a6f76fdf20d48b20, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:04,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:04,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:48:04,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/5ad0d65c56b34997ba7dbba8a0a8fa5c is 50, key is test_row_0/A:col10/1731577684571/Put/seqid=0 2024-11-14T09:48:04,697 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:04,697 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/7bd56576b12c46ffb09c88aa674f1af8 is 50, key is test_row_0/B:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,698 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:04,699 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a51b8d56a1d84933bd1852fb7a69b0f0 is 50, key is test_row_0/A:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741951_1127 (size=14741) 2024-11-14T09:48:04,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741952_1128 (size=13595) 2024-11-14T09:48:04,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/5ad0d65c56b34997ba7dbba8a0a8fa5c 2024-11-14T09:48:04,730 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/7bd56576b12c46ffb09c88aa674f1af8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7bd56576b12c46ffb09c88aa674f1af8 2024-11-14T09:48:04,738 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 7bd56576b12c46ffb09c88aa674f1af8(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:04,738 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:04,738 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577684680; duration=0sec 2024-11-14T09:48:04,738 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:04,738 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:48:04,738 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:04,740 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:04,740 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:48:04,740 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:04,740 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/14dd477313014d3ab8fd7da366a529ba, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.2 K 2024-11-14T09:48:04,741 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 14dd477313014d3ab8fd7da366a529ba, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731577680952 2024-11-14T09:48:04,741 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b47388b7268a4ad4a616411afdb2396c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731577682161 2024-11-14T09:48:04,741 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 024282fc80104ad4ad9b70da03c9fed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:04,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/29a892af61d44f7d8285febd810bba92 is 50, key is test_row_0/B:col10/1731577684571/Put/seqid=0 2024-11-14T09:48:04,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741953_1129 (size=13595) 2024-11-14T09:48:04,773 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/a51b8d56a1d84933bd1852fb7a69b0f0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a51b8d56a1d84933bd1852fb7a69b0f0 2024-11-14T09:48:04,774 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:04,775 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/3833eb1f371e4a6ba1b8b40fcb5fea23 is 50, key is test_row_0/C:col10/1731577683386/Put/seqid=0 2024-11-14T09:48:04,780 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into a51b8d56a1d84933bd1852fb7a69b0f0(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:04,780 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:04,780 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577684680; duration=0sec 2024-11-14T09:48:04,781 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:04,781 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:48:04,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741954_1130 (size=12301) 2024-11-14T09:48:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 389 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577744790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:04,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741955_1131 (size=13595) 2024-11-14T09:48:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 390 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577744894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:05,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 391 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55834 deadline: 1731577745105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:05,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:05,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/29a892af61d44f7d8285febd810bba92 2024-11-14T09:48:05,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d43e7b25f88243f7b77ece7ee1911cd7 is 50, key is test_row_0/C:col10/1731577684571/Put/seqid=0 2024-11-14T09:48:05,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741956_1132 (size=12301) 2024-11-14T09:48:05,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d43e7b25f88243f7b77ece7ee1911cd7 2024-11-14T09:48:05,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/5ad0d65c56b34997ba7dbba8a0a8fa5c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c 2024-11-14T09:48:05,205 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/3833eb1f371e4a6ba1b8b40fcb5fea23 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3833eb1f371e4a6ba1b8b40fcb5fea23 2024-11-14T09:48:05,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c, entries=200, sequenceid=547, filesize=14.4 K 2024-11-14T09:48:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/29a892af61d44f7d8285febd810bba92 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92 2024-11-14T09:48:05,209 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T09:48:05,209 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,211 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into 3833eb1f371e4a6ba1b8b40fcb5fea23(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:05,211 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:05,212 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577684681; duration=0sec 2024-11-14T09:48:05,212 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:05,212 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:48:05,212 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T09:48:05,212 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,213 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T09:48:05,213 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,214 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T09:48:05,214 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,215 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:05,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:05,215 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92, entries=150, sequenceid=547, filesize=12.0 K 2024-11-14T09:48:05,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:05,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:05,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/d43e7b25f88243f7b77ece7ee1911cd7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7 2024-11-14T09:48:05,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7, entries=150, sequenceid=547, filesize=12.0 K 2024-11-14T09:48:05,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 91a5f3dba40c0b5503a8f1cccd94111e in 535ms, sequenceid=547, compaction requested=false 2024-11-14T09:48:05,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:05,418 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:05,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:48:05,419 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T09:48:05,419 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,419 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,419 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:05,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c1033dccedd94caf9d77a9d21b9291a2 is 50, key is test_row_0/A:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:05,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741957_1133 (size=12301) 2024-11-14T09:48:05,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c1033dccedd94caf9d77a9d21b9291a2 2024-11-14T09:48:05,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/41a4bf5499114421be2b03d0dd1d5776 is 50, key is test_row_0/B:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:05,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741958_1134 (size=12301) 2024-11-14T09:48:05,967 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T09:48:05,967 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:05,967 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,967 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:05,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T09:48:06,237 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:06,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/41a4bf5499114421be2b03d0dd1d5776 2024-11-14T09:48:06,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4f536eeaf20845a9be78dd35143ef16a is 50, key is test_row_0/C:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:06,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741959_1135 (size=12301) 2024-11-14T09:48:06,376 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:48:06,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4f536eeaf20845a9be78dd35143ef16a 2024-11-14T09:48:06,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/c1033dccedd94caf9d77a9d21b9291a2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2 2024-11-14T09:48:06,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:48:06,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/41a4bf5499114421be2b03d0dd1d5776 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776 2024-11-14T09:48:06,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:48:06,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/4f536eeaf20845a9be78dd35143ef16a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a 2024-11-14T09:48:06,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a, entries=150, sequenceid=574, filesize=12.0 K 2024-11-14T09:48:06,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 91a5f3dba40c0b5503a8f1cccd94111e in 1277ms, sequenceid=574, compaction requested=true 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:06,696 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 91a5f3dba40c0b5503a8f1cccd94111e:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:06,696 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:06,697 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40637 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:06,697 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:06,698 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/A is initiating minor compaction (all files) 2024-11-14T09:48:06,698 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/B is initiating minor compaction (all files) 2024-11-14T09:48:06,698 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/A in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:06,698 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/B in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:06,698 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a51b8d56a1d84933bd1852fb7a69b0f0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=39.7 K 2024-11-14T09:48:06,698 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7bd56576b12c46ffb09c88aa674f1af8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.3 K 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a51b8d56a1d84933bd1852fb7a69b0f0, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd56576b12c46ffb09c88aa674f1af8, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ad0d65c56b34997ba7dbba8a0a8fa5c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731577684556 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 29a892af61d44f7d8285febd810bba92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731577684567 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1033dccedd94caf9d77a9d21b9291a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731577684769 2024-11-14T09:48:06,699 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a4bf5499114421be2b03d0dd1d5776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731577684769 2024-11-14T09:48:06,707 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:06,707 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#A#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:06,708 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/bdff1b5be1fe498d9cacb428ee7c3479 is 50, key is test_row_0/A:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:06,708 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/09f72154f1e74187a3a5cf8263a6f4d5 is 50, key is test_row_0/B:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:06,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741960_1136 (size=13697) 2024-11-14T09:48:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741961_1137 (size=13697) 2024-11-14T09:48:07,119 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/09f72154f1e74187a3a5cf8263a6f4d5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09f72154f1e74187a3a5cf8263a6f4d5 2024-11-14T09:48:07,122 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/bdff1b5be1fe498d9cacb428ee7c3479 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/bdff1b5be1fe498d9cacb428ee7c3479 2024-11-14T09:48:07,125 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/B of 91a5f3dba40c0b5503a8f1cccd94111e into 09f72154f1e74187a3a5cf8263a6f4d5(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:07,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:07,126 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/B, priority=13, startTime=1731577686696; duration=0sec 2024-11-14T09:48:07,126 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:07,126 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:B 2024-11-14T09:48:07,126 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:07,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:07,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 91a5f3dba40c0b5503a8f1cccd94111e/C is initiating minor compaction (all files) 2024-11-14T09:48:07,127 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 91a5f3dba40c0b5503a8f1cccd94111e/C in TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:07,127 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3833eb1f371e4a6ba1b8b40fcb5fea23, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp, totalSize=37.3 K 2024-11-14T09:48:07,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3833eb1f371e4a6ba1b8b40fcb5fea23, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731577683371 2024-11-14T09:48:07,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d43e7b25f88243f7b77ece7ee1911cd7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1731577684567 2024-11-14T09:48:07,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f536eeaf20845a9be78dd35143ef16a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=574, earliestPutTs=1731577684769 2024-11-14T09:48:07,128 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/A of 91a5f3dba40c0b5503a8f1cccd94111e into bdff1b5be1fe498d9cacb428ee7c3479(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:07,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:07,129 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/A, priority=13, startTime=1731577686696; duration=0sec 2024-11-14T09:48:07,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:07,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:A 2024-11-14T09:48:07,137 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 91a5f3dba40c0b5503a8f1cccd94111e#C#compaction#125 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:07,137 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/f2a3cd9a78ca42ff84223e3cf1c68b5b is 50, key is test_row_0/C:col10/1731577685416/Put/seqid=0 2024-11-14T09:48:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741962_1138 (size=13697) 2024-11-14T09:48:07,547 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/f2a3cd9a78ca42ff84223e3cf1c68b5b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f2a3cd9a78ca42ff84223e3cf1c68b5b 2024-11-14T09:48:07,554 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 91a5f3dba40c0b5503a8f1cccd94111e/C of 91a5f3dba40c0b5503a8f1cccd94111e into f2a3cd9a78ca42ff84223e3cf1c68b5b(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:07,554 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 91a5f3dba40c0b5503a8f1cccd94111e: 2024-11-14T09:48:07,554 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., storeName=91a5f3dba40c0b5503a8f1cccd94111e/C, priority=13, startTime=1731577686696; duration=0sec 2024-11-14T09:48:07,554 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:07,554 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 91a5f3dba40c0b5503a8f1cccd94111e:C 2024-11-14T09:48:07,696 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T09:48:07,696 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:07,696 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:07,696 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:07,696 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:08,700 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T09:48:08,700 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:08,700 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,700 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,701 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:08,755 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T09:48:08,756 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:08,756 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,756 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 317 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4028 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4019 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1846 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5537 rows 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1861 2024-11-14T09:48:08,756 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5582 rows 2024-11-14T09:48:08,757 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:48:08,757 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:48:08,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:08,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:08,760 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:48:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:08,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577688768"}]},"ts":"1731577688768"} 2024-11-14T09:48:08,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:48:08,770 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:48:08,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:48:08,775 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, UNASSIGN}] 2024-11-14T09:48:08,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, UNASSIGN 2024-11-14T09:48:08,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=91a5f3dba40c0b5503a8f1cccd94111e, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:08,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, UNASSIGN because future has completed 2024-11-14T09:48:08,780 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:48:08,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:08,939 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:08,940 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:48:08,941 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 91a5f3dba40c0b5503a8f1cccd94111e, disabling compactions & flushes 2024-11-14T09:48:08,941 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:08,941 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:08,941 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. after waiting 0 ms 2024-11-14T09:48:08,941 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:08,941 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 91a5f3dba40c0b5503a8f1cccd94111e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T09:48:08,949 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f31aaf996dbb409fa0b157ab93d78811 is 50, key is test_row_0/A:col10/1731577685965/Put/seqid=0 2024-11-14T09:48:08,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741963_1139 (size=9857) 2024-11-14T09:48:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:09,356 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f31aaf996dbb409fa0b157ab93d78811 2024-11-14T09:48:09,371 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/7ca5e6fc408f4510b577f9546bde7ffb is 50, key is test_row_0/B:col10/1731577685965/Put/seqid=0 2024-11-14T09:48:09,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741964_1140 (size=9857) 2024-11-14T09:48:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:09,776 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/7ca5e6fc408f4510b577f9546bde7ffb 2024-11-14T09:48:09,784 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c7b8642d383e4303aa5c8ec0896e1e36 is 50, key is test_row_0/C:col10/1731577685965/Put/seqid=0 2024-11-14T09:48:09,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741965_1141 (size=9857) 2024-11-14T09:48:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:10,190 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c7b8642d383e4303aa5c8ec0896e1e36 2024-11-14T09:48:10,196 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/A/f31aaf996dbb409fa0b157ab93d78811 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f31aaf996dbb409fa0b157ab93d78811 2024-11-14T09:48:10,202 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f31aaf996dbb409fa0b157ab93d78811, entries=100, sequenceid=584, filesize=9.6 K 2024-11-14T09:48:10,203 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/B/7ca5e6fc408f4510b577f9546bde7ffb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7ca5e6fc408f4510b577f9546bde7ffb 2024-11-14T09:48:10,208 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7ca5e6fc408f4510b577f9546bde7ffb, entries=100, sequenceid=584, filesize=9.6 K 2024-11-14T09:48:10,209 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/.tmp/C/c7b8642d383e4303aa5c8ec0896e1e36 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c7b8642d383e4303aa5c8ec0896e1e36 2024-11-14T09:48:10,213 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c7b8642d383e4303aa5c8ec0896e1e36, entries=100, sequenceid=584, filesize=9.6 K 2024-11-14T09:48:10,214 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 91a5f3dba40c0b5503a8f1cccd94111e in 1273ms, sequenceid=584, compaction requested=false 2024-11-14T09:48:10,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ee85ac5dd55e4aec8b95371bcea61810, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/57452e2edff945768ecf160b06ad7bd8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cb1034fc6a5443f19a79b24160ef7d8f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a8884f3107e441d4902f52c8ac6e481a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e15cc4efb47141a18ea739633d58021a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ddcb768ecf564d4e95fcae1a6a4879d1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5d2c9c78849e46f484900cd4b029cac4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a7d6663219b4420c8627ecf7f5e57c40, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/8b5678bcd6664c7f82234376af6da5b6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c406eb14c661411691f454a1e9a33b9c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a51b8d56a1d84933bd1852fb7a69b0f0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2] to archive 2024-11-14T09:48:10,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:10,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/3920a6b757d849fcbd2806d16648fc92 2024-11-14T09:48:10,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e40f1c3322454ec9984260f5aa203bf8 2024-11-14T09:48:10,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/77ab293cffe046d39d00ab6fb15e0182 2024-11-14T09:48:10,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ee85ac5dd55e4aec8b95371bcea61810 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ee85ac5dd55e4aec8b95371bcea61810 2024-11-14T09:48:10,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/584970e36f194727b0183f0e488265cb 2024-11-14T09:48:10,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b6b146fd09354dd69c4dc00fc149645c 2024-11-14T09:48:10,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/57452e2edff945768ecf160b06ad7bd8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/57452e2edff945768ecf160b06ad7bd8 2024-11-14T09:48:10,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/aa088fe3249d42eb90c8f543a9df54e2 2024-11-14T09:48:10,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c42e266c879d4081b85818321fde1139 2024-11-14T09:48:10,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f383d1b041e849bdbce02a2dd2579a8e 2024-11-14T09:48:10,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cb1034fc6a5443f19a79b24160ef7d8f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cb1034fc6a5443f19a79b24160ef7d8f 2024-11-14T09:48:10,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/495e8a8358be4be8b118551a4755700a 2024-11-14T09:48:10,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/0226397c7ec24cd780c8dcbdd7106e1d 2024-11-14T09:48:10,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/61abcf73290a4c2082aa17f47c46ce99 2024-11-14T09:48:10,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/fbfa3c4eadfe4685a7ec1a1c8e390b7a 2024-11-14T09:48:10,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/665854445ad443bc8dd206995fa37552 2024-11-14T09:48:10,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/93c9f8299bb34c2c88d9e299840d0f57 2024-11-14T09:48:10,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a8884f3107e441d4902f52c8ac6e481a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a8884f3107e441d4902f52c8ac6e481a 2024-11-14T09:48:10,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b22918d7cede49e8a53fec30a6a3e4f6 2024-11-14T09:48:10,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/387cea94bcd1499bbbb728ed7fb90406 2024-11-14T09:48:10,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e15cc4efb47141a18ea739633d58021a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/e15cc4efb47141a18ea739633d58021a 2024-11-14T09:48:10,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/00874c4a3b5e4d578d91cbecc649d0e3 2024-11-14T09:48:10,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ddcb768ecf564d4e95fcae1a6a4879d1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ddcb768ecf564d4e95fcae1a6a4879d1 2024-11-14T09:48:10,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/79ec7ef792824d5699bcc19979583bae 2024-11-14T09:48:10,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/9f6b43a7c288405e8ea1359ee7c29912 2024-11-14T09:48:10,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ab7546c2829940eea9c6b0824ab14f17 2024-11-14T09:48:10,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5d2c9c78849e46f484900cd4b029cac4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5d2c9c78849e46f484900cd4b029cac4 2024-11-14T09:48:10,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/b02b9f8cb10d475c9f5d9656d389f83a 2024-11-14T09:48:10,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/cd832d6716104608bddbb1ba33b5970a 2024-11-14T09:48:10,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a7d6663219b4420c8627ecf7f5e57c40 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a7d6663219b4420c8627ecf7f5e57c40 2024-11-14T09:48:10,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/06e4bf8b72384f539a61d221a196f52c 2024-11-14T09:48:10,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ae0c927f3db64f48b7a116c1a5b1defe 2024-11-14T09:48:10,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/8b5678bcd6664c7f82234376af6da5b6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/8b5678bcd6664c7f82234376af6da5b6 2024-11-14T09:48:10,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/1d84aede37b146e5801e9cd421460510 2024-11-14T09:48:10,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ebbba27cd5e2465f9184178da039c4ab 2024-11-14T09:48:10,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c406eb14c661411691f454a1e9a33b9c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c406eb14c661411691f454a1e9a33b9c 2024-11-14T09:48:10,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/45b78ada0fc545009bbdfce82936efc4 2024-11-14T09:48:10,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/ed814234c1034d67a6f76fdf20d48b20 2024-11-14T09:48:10,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a51b8d56a1d84933bd1852fb7a69b0f0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/a51b8d56a1d84933bd1852fb7a69b0f0 2024-11-14T09:48:10,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/5ad0d65c56b34997ba7dbba8a0a8fa5c 2024-11-14T09:48:10,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/c1033dccedd94caf9d77a9d21b9291a2 2024-11-14T09:48:10,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bca8113d2bd64094a54e2b9504c7e4e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ad1055ece1e8416fb9760cd64046b928, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09fd49baf8ba40a290cdcd4c3483db17, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4a18be28e5394e648177688fda1890e5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bdb7c3610f534d0c9172b221197aa036, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3c9e29ff5d7f43b88fba8dca45ce8153, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/22b99e6bef6a47deb67b9e9835f29c84, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/971dbde784e341f2b52f1b5a326197c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/f40af7cb414640378029e6c4773e0e10, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/b6724191aab24f72b3df778357ae2181, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/fe4345c726ef47f493d3d3def332c684, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7bd56576b12c46ffb09c88aa674f1af8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776] to archive 2024-11-14T09:48:10,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:10,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3806a45f9ca740ca9d24ddcdd3da5a2a 2024-11-14T09:48:10,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c81347d2da344cf79c9a0b63c5d8e734 2024-11-14T09:48:10,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bca8113d2bd64094a54e2b9504c7e4e7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bca8113d2bd64094a54e2b9504c7e4e7 2024-11-14T09:48:10,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6461a7545ff145fabed8199a109d02b7 2024-11-14T09:48:10,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/6d8d59b765cb479e8f708a4b3c2ac4ca 2024-11-14T09:48:10,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/1f737f140afc44d1b5ba4bbe0d93476c 2024-11-14T09:48:10,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ad1055ece1e8416fb9760cd64046b928 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ad1055ece1e8416fb9760cd64046b928 2024-11-14T09:48:10,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/db8495a616be4682936d46f4b185155f 2024-11-14T09:48:10,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/a12f1e4631584c29ad9fd36cafa9feee 2024-11-14T09:48:10,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09fd49baf8ba40a290cdcd4c3483db17 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09fd49baf8ba40a290cdcd4c3483db17 2024-11-14T09:48:10,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ee1569e6e1fd42749b7120b80bf226d9 2024-11-14T09:48:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/75f2d5d80b78443f8351b2bcbf5242b3 2024-11-14T09:48:10,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/54655689106e4ef1a596c8f31e78ff4a 2024-11-14T09:48:10,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4a18be28e5394e648177688fda1890e5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4a18be28e5394e648177688fda1890e5 2024-11-14T09:48:10,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4460414ec5944f7982d8d4f17462395e 2024-11-14T09:48:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/912023d02d34494783231ead48d2fb05 2024-11-14T09:48:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bdb7c3610f534d0c9172b221197aa036 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/bdb7c3610f534d0c9172b221197aa036 2024-11-14T09:48:10,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/358e0f86485343e7a36ec5096b5dd6c0 2024-11-14T09:48:10,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/dd45fc1abdff4754bd8d84acf3ec0900 2024-11-14T09:48:10,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3c9e29ff5d7f43b88fba8dca45ce8153 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3c9e29ff5d7f43b88fba8dca45ce8153 2024-11-14T09:48:10,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/8c4591e8e670473da1262e7efd3274f9 2024-11-14T09:48:10,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02c573f7538e46828c3a53d96635cca9 2024-11-14T09:48:10,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/22b99e6bef6a47deb67b9e9835f29c84 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/22b99e6bef6a47deb67b9e9835f29c84 2024-11-14T09:48:10,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/e9ea266991234eb5b9e214467f68d388 2024-11-14T09:48:10,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0abdd7f1210341d3a537592271b5ae00 2024-11-14T09:48:10,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/971dbde784e341f2b52f1b5a326197c3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/971dbde784e341f2b52f1b5a326197c3 2024-11-14T09:48:10,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/3e0702c013e2488782c16922cd8ad621 2024-11-14T09:48:10,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/588ee6c1c4b94ff79505ce4f7cfb3924 2024-11-14T09:48:10,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/f40af7cb414640378029e6c4773e0e10 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/f40af7cb414640378029e6c4773e0e10 2024-11-14T09:48:10,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/70b0761b71ee40cfb48383df6db19b9d 2024-11-14T09:48:10,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/ed4668e0e6ee4153b96fa9902c4f267f 2024-11-14T09:48:10,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/b6724191aab24f72b3df778357ae2181 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/b6724191aab24f72b3df778357ae2181 2024-11-14T09:48:10,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/02346820f87d4b748d63564c43b3b149 2024-11-14T09:48:10,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/46424564a81c4d15ae5e4f5bcba3b540 2024-11-14T09:48:10,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/fe4345c726ef47f493d3d3def332c684 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/fe4345c726ef47f493d3d3def332c684 2024-11-14T09:48:10,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/0fb1dbf09c254da7bbf34045f0120c49 2024-11-14T09:48:10,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/4c7fde202fd1429fb8faa1e3c777f498 2024-11-14T09:48:10,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7bd56576b12c46ffb09c88aa674f1af8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7bd56576b12c46ffb09c88aa674f1af8 2024-11-14T09:48:10,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/c7709b237db6459094f81714192d01e4 2024-11-14T09:48:10,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/29a892af61d44f7d8285febd810bba92 2024-11-14T09:48:10,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/41a4bf5499114421be2b03d0dd1d5776 2024-11-14T09:48:10,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/a5338eafed3d4cf2a7845ee3837f8b4e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/993d19ae5dbe483a8267dab180b0f81f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d16abc56cf8b40bfb93be9fd3eee5c7e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/261864a2d0f04485a651d0a17c42be00, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0f5d229cffe049d3b317f3237202c230, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/042aeeeddcde45fca1a8d249f795e48b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/31c7c9ebd59843b3a1670e3ee2969fbe, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c3d6b7218017485fa8a84d0f28056429, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/826c807fe770430d9ab13d898de414eb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/75403b6124594d6cba0f48f3549e3d33, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/14dd477313014d3ab8fd7da366a529ba, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3833eb1f371e4a6ba1b8b40fcb5fea23, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a] to archive 2024-11-14T09:48:10,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:10,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/20dd02cb058c4631ba267a902bba00f3 2024-11-14T09:48:10,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f7166ea9637d486cb6294bbd4815249f 2024-11-14T09:48:10,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/a5338eafed3d4cf2a7845ee3837f8b4e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/a5338eafed3d4cf2a7845ee3837f8b4e 2024-11-14T09:48:10,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5ceb90bcd71042dfb65921afaece8ffe 2024-11-14T09:48:10,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0c5773ce20ed4316ab01791f39b09e12 2024-11-14T09:48:10,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/58ab3eec191e40108596acdef52bace1 2024-11-14T09:48:10,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/993d19ae5dbe483a8267dab180b0f81f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/993d19ae5dbe483a8267dab180b0f81f 2024-11-14T09:48:10,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/7878169c210b4916880a819d2217cf1f 2024-11-14T09:48:10,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/431ca23b990e436ab6c678184431d6e0 2024-11-14T09:48:10,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d16abc56cf8b40bfb93be9fd3eee5c7e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d16abc56cf8b40bfb93be9fd3eee5c7e 2024-11-14T09:48:10,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3d6306fbd76645858429783470ea7644 2024-11-14T09:48:10,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c2f4ee078a2249b3b57b25a464bf7a37 2024-11-14T09:48:10,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d6fc0dbfe33640078ac396db34aaaf46 2024-11-14T09:48:10,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/261864a2d0f04485a651d0a17c42be00 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/261864a2d0f04485a651d0a17c42be00 2024-11-14T09:48:10,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/32632daec7814774a702a77165b0b7f8 2024-11-14T09:48:10,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b567b3d205e6427ebfeca197e562d53c 2024-11-14T09:48:10,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0f5d229cffe049d3b317f3237202c230 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/0f5d229cffe049d3b317f3237202c230 2024-11-14T09:48:10,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/36d89462c24b48bcb4f0af8092fcd1c6 2024-11-14T09:48:10,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/688b699a9de44581a60a19673f17fe9a 2024-11-14T09:48:10,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/042aeeeddcde45fca1a8d249f795e48b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/042aeeeddcde45fca1a8d249f795e48b 2024-11-14T09:48:10,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/cec395c084a74c34a72166d94f32d1c8 2024-11-14T09:48:10,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/37bdca6ca2b74bbd8d31955c19672789 2024-11-14T09:48:10,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/31c7c9ebd59843b3a1670e3ee2969fbe to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/31c7c9ebd59843b3a1670e3ee2969fbe 2024-11-14T09:48:10,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/e3fb7d5952db4d34a46b70500da59861 2024-11-14T09:48:10,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33bb19e04d294e46a217e4a122dd47ad 2024-11-14T09:48:10,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c3d6b7218017485fa8a84d0f28056429 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c3d6b7218017485fa8a84d0f28056429 2024-11-14T09:48:10,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/33e011ee7bec4491aa9380d6b3551031 2024-11-14T09:48:10,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4eae2a79be194348ab8a0b4522d5681f 2024-11-14T09:48:10,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/826c807fe770430d9ab13d898de414eb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/826c807fe770430d9ab13d898de414eb 2024-11-14T09:48:10,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/5247447a11264919bf83d2c861eb2c8c 2024-11-14T09:48:10,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/6ed19d1307124923972660c88970ee68 2024-11-14T09:48:10,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/75403b6124594d6cba0f48f3549e3d33 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/75403b6124594d6cba0f48f3549e3d33 2024-11-14T09:48:10,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/044ec8c635194cd49c00a0ed9ca5b4b2 2024-11-14T09:48:10,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1532ea97e00b4803a5fcca05ea555937 2024-11-14T09:48:10,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/14dd477313014d3ab8fd7da366a529ba to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/14dd477313014d3ab8fd7da366a529ba 2024-11-14T09:48:10,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/1b9036a0fd064192b760634ae734959c 2024-11-14T09:48:10,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/b47388b7268a4ad4a616411afdb2396c 2024-11-14T09:48:10,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3833eb1f371e4a6ba1b8b40fcb5fea23 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/3833eb1f371e4a6ba1b8b40fcb5fea23 2024-11-14T09:48:10,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/024282fc80104ad4ad9b70da03c9fed6 2024-11-14T09:48:10,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/d43e7b25f88243f7b77ece7ee1911cd7 2024-11-14T09:48:10,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/4f536eeaf20845a9be78dd35143ef16a 2024-11-14T09:48:10,379 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/recovered.edits/587.seqid, newMaxSeqId=587, maxSeqId=1 2024-11-14T09:48:10,380 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. 2024-11-14T09:48:10,381 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 91a5f3dba40c0b5503a8f1cccd94111e: Waiting for close lock at 1731577688941Running coprocessor pre-close hooks at 1731577688941Disabling compacts and flushes for region at 1731577688941Disabling writes for close at 1731577688941Obtaining lock to block concurrent updates at 1731577688941Preparing flush snapshotting stores in 91a5f3dba40c0b5503a8f1cccd94111e at 1731577688941Finished memstore snapshotting TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731577688942 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e. at 1731577688943 (+1 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/A: creating writer at 1731577688943Flushing 91a5f3dba40c0b5503a8f1cccd94111e/A: appending metadata at 1731577688949 (+6 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/A: closing flushed file at 1731577688949Flushing 91a5f3dba40c0b5503a8f1cccd94111e/B: creating writer at 1731577689366 (+417 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/B: appending metadata at 1731577689370 (+4 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/B: closing flushed file at 1731577689370Flushing 91a5f3dba40c0b5503a8f1cccd94111e/C: creating writer at 1731577689780 (+410 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/C: appending metadata at 1731577689783 (+3 ms)Flushing 91a5f3dba40c0b5503a8f1cccd94111e/C: closing flushed file at 1731577689783Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b88b861: reopening flushed file at 1731577690195 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52824e93: reopening flushed file at 1731577690202 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@381a4441: reopening flushed file at 1731577690208 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 91a5f3dba40c0b5503a8f1cccd94111e in 1273ms, sequenceid=584, compaction requested=false at 1731577690214 (+6 ms)Writing region close event to WAL at 1731577690376 (+162 ms)Running coprocessor post-close hooks at 1731577690380 (+4 ms)Closed at 1731577690380 2024-11-14T09:48:10,383 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:10,384 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=91a5f3dba40c0b5503a8f1cccd94111e, regionState=CLOSED 2024-11-14T09:48:10,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:10,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T09:48:10,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 91a5f3dba40c0b5503a8f1cccd94111e, server=defc576eb6b7,40243,1731577658084 in 1.6060 sec 2024-11-14T09:48:10,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-14T09:48:10,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=91a5f3dba40c0b5503a8f1cccd94111e, UNASSIGN in 1.6130 sec 2024-11-14T09:48:10,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T09:48:10,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6180 sec 2024-11-14T09:48:10,393 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577690393"}]},"ts":"1731577690393"} 2024-11-14T09:48:10,394 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:48:10,394 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:48:10,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6330 sec 2024-11-14T09:48:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-14T09:48:10,916 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:10,921 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:48:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:10,927 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T09:48:10,929 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:10,933 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:10,939 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/recovered.edits] 2024-11-14T09:48:10,942 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/bdff1b5be1fe498d9cacb428ee7c3479 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/bdff1b5be1fe498d9cacb428ee7c3479 2024-11-14T09:48:10,944 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f31aaf996dbb409fa0b157ab93d78811 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/A/f31aaf996dbb409fa0b157ab93d78811 2024-11-14T09:48:10,947 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09f72154f1e74187a3a5cf8263a6f4d5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/09f72154f1e74187a3a5cf8263a6f4d5 2024-11-14T09:48:10,948 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7ca5e6fc408f4510b577f9546bde7ffb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/B/7ca5e6fc408f4510b577f9546bde7ffb 2024-11-14T09:48:10,950 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c7b8642d383e4303aa5c8ec0896e1e36 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/c7b8642d383e4303aa5c8ec0896e1e36 2024-11-14T09:48:10,952 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f2a3cd9a78ca42ff84223e3cf1c68b5b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/C/f2a3cd9a78ca42ff84223e3cf1c68b5b 2024-11-14T09:48:10,954 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/recovered.edits/587.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e/recovered.edits/587.seqid 2024-11-14T09:48:10,955 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/91a5f3dba40c0b5503a8f1cccd94111e 2024-11-14T09:48:10,955 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:48:10,960 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:10,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T09:48:10,972 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:48:11,003 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:48:11,004 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:11,005 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:48:11,005 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577691005"}]},"ts":"9223372036854775807"} 2024-11-14T09:48:11,010 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:48:11,010 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 91a5f3dba40c0b5503a8f1cccd94111e, NAME => 'TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:48:11,010 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:48:11,011 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577691010"}]},"ts":"9223372036854775807"} 2024-11-14T09:48:11,014 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:48:11,015 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:11,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-14T09:48:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-14T09:48:11,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:48:11,036 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:11,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;defc576eb6b7:40243-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1479602013_22 at /127.0.0.1:33356 [Waiting for operation #289] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1479602013_22 at /127.0.0.1:34114 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-764308505_22 at /127.0.0.1:34220 [Waiting for operation #237] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=430 (was 366) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4375 (was 5082) 2024-11-14T09:48:11,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=430, ProcessCount=11, AvailableMemoryMB=4374 2024-11-14T09:48:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:48:11,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:48:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:11,060 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:48:11,060 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:11,060 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-14T09:48:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:11,061 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:48:11,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741966_1142 (size=959) 2024-11-14T09:48:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:11,472 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:48:11,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741967_1143 (size=53) 2024-11-14T09:48:11,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:11,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:48:11,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2b3f89fc172b9408ca73a69d8ba65061, disabling compactions & flushes 2024-11-14T09:48:11,886 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:11,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:11,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. after waiting 0 ms 2024-11-14T09:48:11,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:11,886 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:11,886 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2b3f89fc172b9408ca73a69d8ba65061: Waiting for close lock at 1731577691885Disabling compacts and flushes for region at 1731577691885Disabling writes for close at 1731577691886 (+1 ms)Writing region close event to WAL at 1731577691886Closed at 1731577691886 2024-11-14T09:48:11,888 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:48:11,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577691889"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577691889"}]},"ts":"1731577691889"} 2024-11-14T09:48:11,894 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:48:11,896 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:48:11,896 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577691896"}]},"ts":"1731577691896"} 2024-11-14T09:48:11,899 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:48:11,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, ASSIGN}] 2024-11-14T09:48:11,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, ASSIGN 2024-11-14T09:48:11,901 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:48:12,052 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:12,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, ASSIGN because future has completed 2024-11-14T09:48:12,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:12,210 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:12,211 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:48:12,211 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,211 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:48:12,211 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,211 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,212 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,214 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName A 2024-11-14T09:48:12,214 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:12,215 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:12,215 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,216 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName B 2024-11-14T09:48:12,216 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:12,217 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:12,217 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,218 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName C 2024-11-14T09:48:12,218 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:12,218 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:12,218 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,219 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,219 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,220 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,220 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,221 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:48:12,222 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,223 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:48:12,224 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 2b3f89fc172b9408ca73a69d8ba65061; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68315352, jitterRate=0.01797807216644287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:48:12,224 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:12,225 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 2b3f89fc172b9408ca73a69d8ba65061: Running coprocessor pre-open hook at 1731577692211Writing region info on filesystem at 1731577692211Initializing all the Stores at 1731577692212 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577692212Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577692212Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577692212Cleaning up temporary data from old regions at 1731577692220 (+8 ms)Running coprocessor post-open hooks at 1731577692224 (+4 ms)Region opened successfully at 1731577692225 (+1 ms) 2024-11-14T09:48:12,225 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., pid=34, masterSystemTime=1731577692207 2024-11-14T09:48:12,227 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:12,227 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:12,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:12,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:12,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T09:48:12,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 in 176 msec 2024-11-14T09:48:12,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-14T09:48:12,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, ASSIGN in 334 msec 2024-11-14T09:48:12,235 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:48:12,235 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577692235"}]},"ts":"1731577692235"} 2024-11-14T09:48:12,236 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:48:12,237 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:48:12,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-14T09:48:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-14T09:48:13,205 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:13,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:13,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:13,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:13,208 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46460dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:13,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:13,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c68170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:13,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:13,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:13,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:48:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:48:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3accb36a 2024-11-14T09:48:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:48:13,219 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:48:13,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:48:13,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:48:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:13,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741968_1144 (size=995) 2024-11-14T09:48:13,642 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T09:48:13,642 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T09:48:13,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:48:13,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE}] 2024-11-14T09:48:13,654 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE 2024-11-14T09:48:13,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:13,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE because future has completed 2024-11-14T09:48:13,657 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:48:13,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:13,810 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:13,810 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:48:13,810 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 2b3f89fc172b9408ca73a69d8ba65061, disabling compactions & flushes 2024-11-14T09:48:13,810 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:13,810 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:13,810 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. after waiting 0 ms 2024-11-14T09:48:13,810 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:13,818 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:48:13,819 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:13,819 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 2b3f89fc172b9408ca73a69d8ba65061: Waiting for close lock at 1731577693810Running coprocessor pre-close hooks at 1731577693810Disabling compacts and flushes for region at 1731577693810Disabling writes for close at 1731577693810Writing region close event to WAL at 1731577693811 (+1 ms)Running coprocessor post-close hooks at 1731577693819 (+8 ms)Closed at 1731577693819 2024-11-14T09:48:13,819 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 2b3f89fc172b9408ca73a69d8ba65061 to self. 2024-11-14T09:48:13,823 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:13,824 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=CLOSED 2024-11-14T09:48:13,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:13,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-14T09:48:13,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 in 171 msec 2024-11-14T09:48:13,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=true 2024-11-14T09:48:13,982 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:13,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE because future has completed 2024-11-14T09:48:13,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:14,151 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,151 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:48:14,152 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,152 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:48:14,153 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,153 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,159 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,164 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName A 2024-11-14T09:48:14,166 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:14,167 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:14,167 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,168 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName B 2024-11-14T09:48:14,168 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:14,169 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:14,169 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,169 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b3f89fc172b9408ca73a69d8ba65061 columnFamilyName C 2024-11-14T09:48:14,169 DEBUG [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:14,170 INFO [StoreOpener-2b3f89fc172b9408ca73a69d8ba65061-1 {}] regionserver.HStore(327): Store=2b3f89fc172b9408ca73a69d8ba65061/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:14,170 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,170 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,171 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,172 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,172 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,173 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:48:14,174 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,174 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2b3f89fc172b9408ca73a69d8ba65061; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61517475, jitterRate=-0.08331818878650665}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:48:14,175 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,177 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2b3f89fc172b9408ca73a69d8ba65061: Running coprocessor pre-open hook at 1731577694153Writing region info on filesystem at 1731577694153Initializing all the Stores at 1731577694155 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577694155Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577694156 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577694156Cleaning up temporary data from old regions at 1731577694172 (+16 ms)Running coprocessor post-open hooks at 1731577694175 (+3 ms)Region opened successfully at 1731577694177 (+2 ms) 2024-11-14T09:48:14,177 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., pid=39, masterSystemTime=1731577694142 2024-11-14T09:48:14,179 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,180 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,180 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:14,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-14T09:48:14,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 in 194 msec 2024-11-14T09:48:14,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T09:48:14,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, REOPEN/MOVE in 531 msec 2024-11-14T09:48:14,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T09:48:14,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-14T09:48:14,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 964 msec 2024-11-14T09:48:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-14T09:48:14,193 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:14,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,194 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6aa8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,196 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,198 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66cbe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,202 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f1b4e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,203 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735f655c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b57374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,207 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e1821a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,209 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338d0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,210 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8a06ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,213 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c728dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b91114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,217 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f811083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341602f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,220 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,222 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57bfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:14,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:14,225 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714d355d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:14,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:14,227 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:14,228 DEBUG [Thread-724 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,228 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,228 DEBUG [Thread-715 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,229 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,230 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,230 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T09:48:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,232 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,233 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,234 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:14,234 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:14,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,236 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:14,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:48:14,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:14,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114defd3a9a8ead4fbd851e36c3635931b2_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577694244/Put/seqid=0 2024-11-14T09:48:14,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741969_1145 (size=19474) 2024-11-14T09:48:14,297 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:14,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114defd3a9a8ead4fbd851e36c3635931b2_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114defd3a9a8ead4fbd851e36c3635931b2_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/0b90be113db84cd589fce9cf159d34c3, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:14,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/0b90be113db84cd589fce9cf159d34c3 is 175, key is test_row_0/A:col10/1731577694244/Put/seqid=0 2024-11-14T09:48:14,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741970_1146 (size=56733) 2024-11-14T09:48:14,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577754310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577754312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577754314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577754324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577754325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:14,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577754434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577754435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577754435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577754437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577754437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:14,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577754645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577754646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577754646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577754646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577754646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:14,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:14,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:14,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:14,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/0b90be113db84cd589fce9cf159d34c3 2024-11-14T09:48:14,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/728dd5e467494befbcf2be5e83b78ff8 is 50, key is test_row_0/B:col10/1731577694244/Put/seqid=0 2024-11-14T09:48:14,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741971_1147 (size=12001) 2024-11-14T09:48:14,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/728dd5e467494befbcf2be5e83b78ff8 2024-11-14T09:48:14,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8f5f45f80c3f4a518a36cb92e31576ed is 50, key is test_row_0/C:col10/1731577694244/Put/seqid=0 2024-11-14T09:48:14,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:14,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:14,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:14,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741972_1148 (size=12001) 2024-11-14T09:48:14,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8f5f45f80c3f4a518a36cb92e31576ed 2024-11-14T09:48:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/0b90be113db84cd589fce9cf159d34c3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3 2024-11-14T09:48:14,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3, entries=300, sequenceid=16, filesize=55.4 K 2024-11-14T09:48:14,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/728dd5e467494befbcf2be5e83b78ff8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8 2024-11-14T09:48:14,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T09:48:14,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8f5f45f80c3f4a518a36cb92e31576ed as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed 2024-11-14T09:48:14,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T09:48:14,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2b3f89fc172b9408ca73a69d8ba65061 in 660ms, sequenceid=16, compaction requested=false 2024-11-14T09:48:14,910 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T09:48:14,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:14,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:14,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:48:15,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577754996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577754996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577754996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577754998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577754999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c57388bd90b449390178031a4103ee3_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577694965/Put/seqid=0 2024-11-14T09:48:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741973_1149 (size=12154) 2024-11-14T09:48:15,054 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:15,059 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145c57388bd90b449390178031a4103ee3_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c57388bd90b449390178031a4103ee3_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:15,060 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/f0065c541efb41fa859b7b3d64df0bd0, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:15,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/f0065c541efb41fa859b7b3d64df0bd0 is 175, key is test_row_0/A:col10/1731577694965/Put/seqid=0 2024-11-14T09:48:15,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741974_1150 (size=30955) 2024-11-14T09:48:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577755115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577755115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577755115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577755116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577755116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577755324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577755324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577755325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577755325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577755326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:15,467 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=43, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/f0065c541efb41fa859b7b3d64df0bd0 2024-11-14T09:48:15,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/732ba0ef15f04faaa96a6da64d517f65 is 50, key is test_row_0/B:col10/1731577694965/Put/seqid=0 2024-11-14T09:48:15,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741975_1151 (size=12001) 2024-11-14T09:48:15,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577755634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577755635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577755635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577755635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577755635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:15,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/732ba0ef15f04faaa96a6da64d517f65 2024-11-14T09:48:15,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1f9af4e2e73143d5804146e715d2c72b is 50, key is test_row_0/C:col10/1731577694965/Put/seqid=0 2024-11-14T09:48:15,935 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:48:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741976_1152 (size=12001) 2024-11-14T09:48:15,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:15,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:15,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:15,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:16,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577756146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:16,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577756146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:16,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577756146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:16,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577756155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577756156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:16,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:16,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1f9af4e2e73143d5804146e715d2c72b 2024-11-14T09:48:16,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/f0065c541efb41fa859b7b3d64df0bd0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0 2024-11-14T09:48:16,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0, entries=150, sequenceid=43, filesize=30.2 K 2024-11-14T09:48:16,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/732ba0ef15f04faaa96a6da64d517f65 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65 2024-11-14T09:48:16,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T09:48:16,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1f9af4e2e73143d5804146e715d2c72b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b 2024-11-14T09:48:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:16,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b, entries=150, sequenceid=43, filesize=11.7 K 2024-11-14T09:48:16,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 2b3f89fc172b9408ca73a69d8ba65061 in 1430ms, sequenceid=43, compaction requested=false 2024-11-14T09:48:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:16,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T09:48:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,412 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T09:48:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ca499827b1546529e923aebb55e5a81_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577694995/Put/seqid=0 2024-11-14T09:48:16,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741977_1153 (size=12154) 2024-11-14T09:48:16,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,456 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ca499827b1546529e923aebb55e5a81_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ca499827b1546529e923aebb55e5a81_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:16,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/2511e673946648ee83c71670a410c5ff, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:16,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/2511e673946648ee83c71670a410c5ff is 175, key is test_row_0/A:col10/1731577694995/Put/seqid=0 2024-11-14T09:48:16,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741978_1154 (size=30955) 2024-11-14T09:48:16,465 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/2511e673946648ee83c71670a410c5ff 2024-11-14T09:48:16,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/00e68a52a5994ba8ab41266372538b51 is 50, key is test_row_0/B:col10/1731577694995/Put/seqid=0 2024-11-14T09:48:16,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741979_1155 (size=12001) 2024-11-14T09:48:16,483 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/00e68a52a5994ba8ab41266372538b51 2024-11-14T09:48:16,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1587b5250f7642a4a5b50e0b6c6474d6 is 50, key is test_row_0/C:col10/1731577694995/Put/seqid=0 2024-11-14T09:48:16,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741980_1156 (size=12001) 2024-11-14T09:48:16,930 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1587b5250f7642a4a5b50e0b6c6474d6 2024-11-14T09:48:16,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/2511e673946648ee83c71670a410c5ff as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff 2024-11-14T09:48:16,940 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff, entries=150, sequenceid=53, filesize=30.2 K 2024-11-14T09:48:16,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/00e68a52a5994ba8ab41266372538b51 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51 2024-11-14T09:48:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,953 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T09:48:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1587b5250f7642a4a5b50e0b6c6474d6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6 2024-11-14T09:48:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,961 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6, entries=150, sequenceid=53, filesize=11.7 K 2024-11-14T09:48:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,963 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 2b3f89fc172b9408ca73a69d8ba65061 in 551ms, sequenceid=53, compaction requested=true 2024-11-14T09:48:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:16,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T09:48:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T09:48:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T09:48:16,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7290 sec 2024-11-14T09:48:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.7390 sec 2024-11-14T09:48:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:17,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:17,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114503e28e18dc24ca18e2e6c6b22bb4c09_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577757263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577757265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741982_1158 (size=24358) 2024-11-14T09:48:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577757267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577757267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577757268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577757375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577757375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577757384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577757384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577757384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577757585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577757586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577757595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577757595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577757595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,675 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:17,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114503e28e18dc24ca18e2e6c6b22bb4c09_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114503e28e18dc24ca18e2e6c6b22bb4c09_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:17,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/37cd26fbcb6c4c3bb469a918cf4a4606, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:17,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/37cd26fbcb6c4c3bb469a918cf4a4606 is 175, key is test_row_0/A:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741981_1157 (size=73995) 2024-11-14T09:48:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577757895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577757895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577757904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577757905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577757905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/37cd26fbcb6c4c3bb469a918cf4a4606 2024-11-14T09:48:18,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/c6ea5a27ae0c46de93857fe0f2177837 is 50, key is test_row_0/B:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:18,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741983_1159 (size=12001) 2024-11-14T09:48:18,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/c6ea5a27ae0c46de93857fe0f2177837 2024-11-14T09:48:18,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/37d17bc98ad24114ba1989f457a7d139 is 50, key is test_row_0/C:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:18,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741984_1160 (size=12001) 2024-11-14T09:48:18,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/37d17bc98ad24114ba1989f457a7d139 2024-11-14T09:48:18,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/37cd26fbcb6c4c3bb469a918cf4a4606 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606 2024-11-14T09:48:18,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606, entries=400, sequenceid=64, filesize=72.3 K 2024-11-14T09:48:18,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/c6ea5a27ae0c46de93857fe0f2177837 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837 2024-11-14T09:48:18,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837, entries=150, sequenceid=64, filesize=11.7 K 2024-11-14T09:48:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/37d17bc98ad24114ba1989f457a7d139 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139 2024-11-14T09:48:18,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139, entries=150, sequenceid=64, filesize=11.7 K 2024-11-14T09:48:18,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2b3f89fc172b9408ca73a69d8ba65061 in 953ms, sequenceid=64, compaction requested=true 2024-11-14T09:48:18,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:18,172 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:18,173 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:18,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:18,176 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:48:18,176 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 192638 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:48:18,176 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:18,176 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:18,176 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,176 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,176 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=46.9 K 2024-11-14T09:48:18,176 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=188.1 K 2024-11-14T09:48:18,176 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,177 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 728dd5e467494befbcf2be5e83b78ff8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577694244 2024-11-14T09:48:18,178 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606] 2024-11-14T09:48:18,179 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 732ba0ef15f04faaa96a6da64d517f65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731577694308 2024-11-14T09:48:18,179 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b90be113db84cd589fce9cf159d34c3, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577694243 2024-11-14T09:48:18,179 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 00e68a52a5994ba8ab41266372538b51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731577694991 2024-11-14T09:48:18,180 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0065c541efb41fa859b7b3d64df0bd0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731577694308 2024-11-14T09:48:18,180 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ea5a27ae0c46de93857fe0f2177837, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697212 2024-11-14T09:48:18,181 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2511e673946648ee83c71670a410c5ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731577694991 2024-11-14T09:48:18,181 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37cd26fbcb6c4c3bb469a918cf4a4606, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697168 2024-11-14T09:48:18,209 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:18,213 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:18,214 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/81529dce73014c569f382df14e746f1e is 50, key is test_row_0/B:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:18,220 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ac88f14c25ac4769a26f7ccdd0c8d812_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:18,226 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ac88f14c25ac4769a26f7ccdd0c8d812_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:18,227 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ac88f14c25ac4769a26f7ccdd0c8d812_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:18,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741985_1161 (size=12139) 2024-11-14T09:48:18,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741986_1162 (size=4469) 2024-11-14T09:48:18,249 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/81529dce73014c569f382df14e746f1e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/81529dce73014c569f382df14e746f1e 2024-11-14T09:48:18,257 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into 81529dce73014c569f382df14e746f1e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:18,258 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:18,258 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=12, startTime=1731577698173; duration=0sec 2024-11-14T09:48:18,258 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:18,258 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:18,258 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:48:18,261 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:48:18,261 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:18,262 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,262 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=46.9 K 2024-11-14T09:48:18,262 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5f45f80c3f4a518a36cb92e31576ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577694244 2024-11-14T09:48:18,263 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9af4e2e73143d5804146e715d2c72b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731577694308 2024-11-14T09:48:18,264 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1587b5250f7642a4a5b50e0b6c6474d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731577694991 2024-11-14T09:48:18,264 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d17bc98ad24114ba1989f457a7d139, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697212 2024-11-14T09:48:18,275 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:18,276 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/0cba974a549d442ca3f1ca31d007f639 is 50, key is test_row_0/C:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:18,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741987_1163 (size=12139) 2024-11-14T09:48:18,295 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/0cba974a549d442ca3f1ca31d007f639 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/0cba974a549d442ca3f1ca31d007f639 2024-11-14T09:48:18,303 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 0cba974a549d442ca3f1ca31d007f639(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:18,303 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:18,303 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=12, startTime=1731577698173; duration=0sec 2024-11-14T09:48:18,304 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:18,304 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T09:48:18,411 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:18,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:18,414 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:18,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T09:48:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T09:48:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:18,415 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:18,416 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:18,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:18,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577758425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577758425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e8b2084aab34a0297f08a4fee7716ef_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577698405/Put/seqid=0 2024-11-14T09:48:18,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577758435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577758438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577758446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741988_1164 (size=14594) 2024-11-14T09:48:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:18,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577758554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577758554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577758565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:18,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:18,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,642 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#141 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:18,643 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/c20111374b3442dd970a704f8bbfbd93 is 175, key is test_row_0/A:col10/1731577697212/Put/seqid=0 2024-11-14T09:48:18,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741989_1165 (size=31093) 2024-11-14T09:48:18,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577758764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577758764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577758775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:18,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:18,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:18,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:18,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:18,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:18,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:18,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e8b2084aab34a0297f08a4fee7716ef_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e8b2084aab34a0297f08a4fee7716ef_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:18,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5aad033fd2524318824cf58f2f98ca0e, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:18,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5aad033fd2524318824cf58f2f98ca0e is 175, key is test_row_0/A:col10/1731577698405/Put/seqid=0 2024-11-14T09:48:18,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741990_1166 (size=39549) 2024-11-14T09:48:19,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:19,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/c20111374b3442dd970a704f8bbfbd93 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93 2024-11-14T09:48:19,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577759075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577759085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,085 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into c20111374b3442dd970a704f8bbfbd93(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:19,085 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:19,085 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=12, startTime=1731577698172; duration=0sec 2024-11-14T09:48:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,086 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:19,086 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577759085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5aad033fd2524318824cf58f2f98ca0e 2024-11-14T09:48:19,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4af83460973e42de885168fd696b3d2f is 50, key is test_row_0/B:col10/1731577698405/Put/seqid=0 2024-11-14T09:48:19,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741991_1167 (size=12001) 2024-11-14T09:48:19,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4af83460973e42de885168fd696b3d2f 2024-11-14T09:48:19,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/21297aa82b0b43a4b4aa7b155b986e51 is 50, key is test_row_0/C:col10/1731577698405/Put/seqid=0 2024-11-14T09:48:19,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741992_1168 (size=12001) 2024-11-14T09:48:19,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/21297aa82b0b43a4b4aa7b155b986e51 2024-11-14T09:48:19,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5aad033fd2524318824cf58f2f98ca0e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e 2024-11-14T09:48:19,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e, entries=200, sequenceid=93, filesize=38.6 K 2024-11-14T09:48:19,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4af83460973e42de885168fd696b3d2f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f 2024-11-14T09:48:19,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T09:48:19,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/21297aa82b0b43a4b4aa7b155b986e51 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51 2024-11-14T09:48:19,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51, entries=150, sequenceid=93, filesize=11.7 K 2024-11-14T09:48:19,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 2b3f89fc172b9408ca73a69d8ba65061 in 935ms, sequenceid=93, compaction requested=false 2024-11-14T09:48:19,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:19,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:19,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d18d57760b084bd1ad722a63ffb51201_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:19,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741993_1169 (size=14594) 2024-11-14T09:48:19,487 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:19,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d18d57760b084bd1ad722a63ffb51201_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d18d57760b084bd1ad722a63ffb51201_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:19,497 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/80a006bb51494bf88b663188997f1b71, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:19,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/80a006bb51494bf88b663188997f1b71 is 175, key is test_row_0/A:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741994_1170 (size=39549) 2024-11-14T09:48:19,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/80a006bb51494bf88b663188997f1b71 2024-11-14T09:48:19,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577759517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/1f6144c1b7554a4ea17914ab643eca87 is 50, key is test_row_0/B:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:19,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577759523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741995_1171 (size=12001) 2024-11-14T09:48:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:19,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577759585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577759594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577759594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577759635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577759635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577759845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577759845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:19,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:19,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/1f6144c1b7554a4ea17914ab643eca87 2024-11-14T09:48:19,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:19,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:19,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a8c518edcb1b4630bc38ae270da31a1f is 50, key is test_row_0/C:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:20,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741996_1172 (size=12001) 2024-11-14T09:48:20,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a8c518edcb1b4630bc38ae270da31a1f 2024-11-14T09:48:20,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/80a006bb51494bf88b663188997f1b71 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71 2024-11-14T09:48:20,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71, entries=200, sequenceid=105, filesize=38.6 K 2024-11-14T09:48:20,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/1f6144c1b7554a4ea17914ab643eca87 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87 2024-11-14T09:48:20,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T09:48:20,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a8c518edcb1b4630bc38ae270da31a1f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f 2024-11-14T09:48:20,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T09:48:20,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2b3f89fc172b9408ca73a69d8ba65061 in 582ms, sequenceid=105, compaction requested=true 2024-11-14T09:48:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:20,030 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:20,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:20,031 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:20,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:20,032 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:20,033 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:20,033 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:20,033 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/81529dce73014c569f382df14e746f1e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.3 K 2024-11-14T09:48:20,033 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:20,033 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:20,033 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:20,033 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=107.6 K 2024-11-14T09:48:20,033 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:20,034 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71] 2024-11-14T09:48:20,035 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20111374b3442dd970a704f8bbfbd93, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697212 2024-11-14T09:48:20,035 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 81529dce73014c569f382df14e746f1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697212 2024-11-14T09:48:20,035 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aad033fd2524318824cf58f2f98ca0e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731577697260 2024-11-14T09:48:20,036 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af83460973e42de885168fd696b3d2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731577697260 2024-11-14T09:48:20,036 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80a006bb51494bf88b663188997f1b71, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:20,036 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f6144c1b7554a4ea17914ab643eca87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:20,053 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:20,066 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:20,067 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/2758b86e2baa47918c273cf3e5a66b0b is 50, key is test_row_0/B:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:20,076 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149d92151827db4d7cba8762629049db6a_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:20,079 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149d92151827db4d7cba8762629049db6a_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:20,079 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d92151827db4d7cba8762629049db6a_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:20,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T09:48:20,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:20,105 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:48:20,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741997_1173 (size=12241) 2024-11-14T09:48:20,114 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/2758b86e2baa47918c273cf3e5a66b0b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/2758b86e2baa47918c273cf3e5a66b0b 2024-11-14T09:48:20,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741998_1174 (size=4469) 2024-11-14T09:48:20,123 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#150 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:20,124 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/de86ab0476744378b800477bf97d69cb is 175, key is test_row_0/A:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:20,124 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into 2758b86e2baa47918c273cf3e5a66b0b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:20,124 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:20,124 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577700030; duration=0sec 2024-11-14T09:48:20,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:20,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:20,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:20,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:20,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:20,127 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:20,127 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/0cba974a549d442ca3f1ca31d007f639, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.3 K 2024-11-14T09:48:20,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cba974a549d442ca3f1ca31d007f639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731577697212 2024-11-14T09:48:20,130 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 21297aa82b0b43a4b4aa7b155b986e51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731577697260 2024-11-14T09:48:20,131 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c518edcb1b4630bc38ae270da31a1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:20,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145eb5887974504b4692afeffa0c50be0b_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577699505/Put/seqid=0 2024-11-14T09:48:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:20,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:20,165 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:20,166 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/e704e9d12daa4b478d3f4c29c4e3cfb0 is 50, key is test_row_0/C:col10/1731577698430/Put/seqid=0 2024-11-14T09:48:20,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741999_1175 (size=31195) 2024-11-14T09:48:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577760173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577760175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,180 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/de86ab0476744378b800477bf97d69cb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb 2024-11-14T09:48:20,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742000_1176 (size=12204) 2024-11-14T09:48:20,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:20,189 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into de86ab0476744378b800477bf97d69cb(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:20,189 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:20,189 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577700030; duration=0sec 2024-11-14T09:48:20,189 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:20,191 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:20,194 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145eb5887974504b4692afeffa0c50be0b_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145eb5887974504b4692afeffa0c50be0b_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/9fea2f6765c949449f3485b061f83b83, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/9fea2f6765c949449f3485b061f83b83 is 175, key is test_row_0/A:col10/1731577699505/Put/seqid=0 2024-11-14T09:48:20,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742001_1177 (size=12241) 2024-11-14T09:48:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742002_1178 (size=31005) 2024-11-14T09:48:20,222 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/e704e9d12daa4b478d3f4c29c4e3cfb0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/e704e9d12daa4b478d3f4c29c4e3cfb0 2024-11-14T09:48:20,227 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into e704e9d12daa4b478d3f4c29c4e3cfb0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:20,227 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:20,227 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577700031; duration=0sec 2024-11-14T09:48:20,227 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:20,228 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577760284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577760284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577760495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577760495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577760604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577760604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,605 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577760604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,605 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,606 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,623 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/9fea2f6765c949449f3485b061f83b83 2024-11-14T09:48:20,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/8e305e5197a14d9b8e969ac5b5b9060b is 50, key is test_row_0/B:col10/1731577699505/Put/seqid=0 2024-11-14T09:48:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742003_1179 (size=12051) 2024-11-14T09:48:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577760805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577760805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:21,036 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/8e305e5197a14d9b8e969ac5b5b9060b 2024-11-14T09:48:21,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/c74391fd3c0a466180dea0d6abbef124 is 50, key is test_row_0/C:col10/1731577699505/Put/seqid=0 2024-11-14T09:48:21,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742004_1180 (size=12051) 2024-11-14T09:48:21,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577761315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:21,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577761316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:21,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:21,447 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/c74391fd3c0a466180dea0d6abbef124 2024-11-14T09:48:21,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/9fea2f6765c949449f3485b061f83b83 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83 2024-11-14T09:48:21,457 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83, entries=150, sequenceid=130, filesize=30.3 K 2024-11-14T09:48:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/8e305e5197a14d9b8e969ac5b5b9060b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b 2024-11-14T09:48:21,464 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T09:48:21,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/c74391fd3c0a466180dea0d6abbef124 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124 2024-11-14T09:48:21,470 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T09:48:21,471 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2b3f89fc172b9408ca73a69d8ba65061 in 1367ms, sequenceid=130, compaction requested=false 2024-11-14T09:48:21,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:21,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:21,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T09:48:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T09:48:21,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T09:48:21,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0570 sec 2024-11-14T09:48:21,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 3.0640 sec 2024-11-14T09:48:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:22,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T09:48:22,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149f870657a68543ea833861e935478672_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:22,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742005_1181 (size=14794) 2024-11-14T09:48:22,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577762380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577762384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577762494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577762494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T09:48:22,575 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:22,576 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T09:48:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:22,578 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:22,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:22,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577762615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577762615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,617 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,618 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:22,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577762625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,626 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:22,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577762705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577762705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:22,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:22,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:48:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:22,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149f870657a68543ea833861e935478672_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f870657a68543ea833861e935478672_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:22,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/fdbea4a1d20c46f19f8a22b61c8e6ff8, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:22,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 is 175, key is test_row_0/A:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:22,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742006_1182 (size=39749) 2024-11-14T09:48:22,779 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 2024-11-14T09:48:22,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b05a6c8b8b1a4f209a3b90c2e43eef4e is 50, key is test_row_0/B:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:22,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742007_1183 (size=12151) 2024-11-14T09:48:22,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b05a6c8b8b1a4f209a3b90c2e43eef4e 2024-11-14T09:48:22,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/14cbd3ace9d94f28b4cce7e9b0995736 is 50, key is test_row_0/C:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:22,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742008_1184 (size=12151) 2024-11-14T09:48:22,880 INFO [master/defc576eb6b7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T09:48:22,881 INFO [master/defc576eb6b7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T09:48:22,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:48:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:22,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:23,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577763015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577763015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:48:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:48:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:23,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/14cbd3ace9d94f28b4cce7e9b0995736 2024-11-14T09:48:23,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 2024-11-14T09:48:23,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8, entries=200, sequenceid=145, filesize=38.8 K 2024-11-14T09:48:23,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b05a6c8b8b1a4f209a3b90c2e43eef4e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e 2024-11-14T09:48:23,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:48:23,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/14cbd3ace9d94f28b4cce7e9b0995736 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736 2024-11-14T09:48:23,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736, entries=150, sequenceid=145, filesize=11.9 K 2024-11-14T09:48:23,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2b3f89fc172b9408ca73a69d8ba65061 in 919ms, sequenceid=145, compaction requested=true 2024-11-14T09:48:23,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:23,248 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:23,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:23,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:23,248 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:23,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:23,249 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:23,249 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:23,249 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,250 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=99.6 K 2024-11-14T09:48:23,250 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,250 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8] 2024-11-14T09:48:23,250 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:23,250 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting de86ab0476744378b800477bf97d69cb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:23,250 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:23,250 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,250 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/2758b86e2baa47918c273cf3e5a66b0b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.6 K 2024-11-14T09:48:23,251 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 2758b86e2baa47918c273cf3e5a66b0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:23,251 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fea2f6765c949449f3485b061f83b83, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577699505 2024-11-14T09:48:23,251 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e305e5197a14d9b8e969ac5b5b9060b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577699505 2024-11-14T09:48:23,251 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdbea4a1d20c46f19f8a22b61c8e6ff8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700156 2024-11-14T09:48:23,251 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b05a6c8b8b1a4f209a3b90c2e43eef4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700165 2024-11-14T09:48:23,266 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:23,266 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:23,267 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/1e6274136a9649f3a64c20c5a1e231b9 is 50, key is test_row_0/B:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:23,269 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411141bb423b34740420d8638e7b60fc4cbdb_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:23,271 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411141bb423b34740420d8638e7b60fc4cbdb_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:23,271 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141bb423b34740420d8638e7b60fc4cbdb_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:23,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742009_1185 (size=12493) 2024-11-14T09:48:23,312 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/1e6274136a9649f3a64c20c5a1e231b9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1e6274136a9649f3a64c20c5a1e231b9 2024-11-14T09:48:23,320 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into 1e6274136a9649f3a64c20c5a1e231b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:23,320 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:23,320 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577703248; duration=0sec 2024-11-14T09:48:23,320 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:23,320 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:23,320 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:23,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742010_1186 (size=4469) 2024-11-14T09:48:23,323 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:23,323 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:23,323 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,323 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/e704e9d12daa4b478d3f4c29c4e3cfb0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.6 K 2024-11-14T09:48:23,327 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e704e9d12daa4b478d3f4c29c4e3cfb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731577698430 2024-11-14T09:48:23,327 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#160 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:23,328 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/7fef9db6bc234ad3a26ca8898106529c is 175, key is test_row_0/A:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:23,330 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c74391fd3c0a466180dea0d6abbef124, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731577699505 2024-11-14T09:48:23,331 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 14cbd3ace9d94f28b4cce7e9b0995736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700165 2024-11-14T09:48:23,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T09:48:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:23,349 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:48:23,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742011_1187 (size=31447) 2024-11-14T09:48:23,356 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#161 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:23,356 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/27ab80e0fc7d455380efb6a4d7221213 is 50, key is test_row_0/C:col10/1731577700165/Put/seqid=0 2024-11-14T09:48:23,363 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/7fef9db6bc234ad3a26ca8898106529c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c 2024-11-14T09:48:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111462c5330af84447fdb258e74c983909d0_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577702374/Put/seqid=0 2024-11-14T09:48:23,374 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into 7fef9db6bc234ad3a26ca8898106529c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:23,374 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:23,375 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577703248; duration=0sec 2024-11-14T09:48:23,375 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:23,375 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:23,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742012_1188 (size=12493) 2024-11-14T09:48:23,395 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/27ab80e0fc7d455380efb6a4d7221213 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/27ab80e0fc7d455380efb6a4d7221213 2024-11-14T09:48:23,405 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 27ab80e0fc7d455380efb6a4d7221213(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:23,405 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:23,405 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577703249; duration=0sec 2024-11-14T09:48:23,405 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:23,405 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:23,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742013_1189 (size=12304) 2024-11-14T09:48:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:23,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:23,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577763575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577763575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577763685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577763685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:23,813 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111462c5330af84447fdb258e74c983909d0_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462c5330af84447fdb258e74c983909d0_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:23,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/22b7429b9e5142aeaba135aed215aa6d, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:23,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/22b7429b9e5142aeaba135aed215aa6d is 175, key is test_row_0/A:col10/1731577702374/Put/seqid=0 2024-11-14T09:48:23,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742014_1190 (size=31105) 2024-11-14T09:48:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577763894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577763894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577764205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577764205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,220 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/22b7429b9e5142aeaba135aed215aa6d 2024-11-14T09:48:24,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/70077198e23e4106bd5708453a7df29d is 50, key is test_row_0/B:col10/1731577702374/Put/seqid=0 2024-11-14T09:48:24,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742015_1191 (size=12151) 2024-11-14T09:48:24,233 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/70077198e23e4106bd5708453a7df29d 2024-11-14T09:48:24,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/90089d8e23de4c9dbadb96bb0fefd641 is 50, key is test_row_0/C:col10/1731577702374/Put/seqid=0 2024-11-14T09:48:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742016_1192 (size=12151) 2024-11-14T09:48:24,646 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/90089d8e23de4c9dbadb96bb0fefd641 2024-11-14T09:48:24,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/22b7429b9e5142aeaba135aed215aa6d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d 2024-11-14T09:48:24,661 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d, entries=150, sequenceid=170, filesize=30.4 K 2024-11-14T09:48:24,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/70077198e23e4106bd5708453a7df29d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d 2024-11-14T09:48:24,668 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T09:48:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/90089d8e23de4c9dbadb96bb0fefd641 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641 2024-11-14T09:48:24,676 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T09:48:24,678 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2b3f89fc172b9408ca73a69d8ba65061 in 1329ms, sequenceid=170, compaction requested=false 2024-11-14T09:48:24,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:24,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:24,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T09:48:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T09:48:24,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T09:48:24,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-14T09:48:24,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.1090 sec 2024-11-14T09:48:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:24,717 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:24,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:48:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T09:48:24,725 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:24,725 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T09:48:24,728 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:24,729 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:24,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:24,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111452e4fdb9e8d14d48b62b44af8834437b_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:24,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742017_1193 (size=17284) 2024-11-14T09:48:24,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577764808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577764810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:24,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:24,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:24,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:24,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:24,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577764915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577764925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:25,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577765124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577765135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,175 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:25,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111452e4fdb9e8d14d48b62b44af8834437b_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452e4fdb9e8d14d48b62b44af8834437b_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:25,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ad76122b493b464eb2ad5aac8e9f83ef, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:25,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ad76122b493b464eb2ad5aac8e9f83ef is 175, key is test_row_0/A:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:25,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742018_1194 (size=48389) 2024-11-14T09:48:25,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:25,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577765435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577765444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,599 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ad76122b493b464eb2ad5aac8e9f83ef 2024-11-14T09:48:25,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/602c8f83057445debd22bd2b11a8e950 is 50, key is test_row_0/B:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:25,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742019_1195 (size=12151) 2024-11-14T09:48:25,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/602c8f83057445debd22bd2b11a8e950 2024-11-14T09:48:25,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 is 50, key is test_row_0/C:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742020_1196 (size=12151) 2024-11-14T09:48:25,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:25,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577765945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577765955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:25,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:25,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:26,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 2024-11-14T09:48:26,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ad76122b493b464eb2ad5aac8e9f83ef as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef 2024-11-14T09:48:26,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef, entries=250, sequenceid=186, filesize=47.3 K 2024-11-14T09:48:26,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/602c8f83057445debd22bd2b11a8e950 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950 2024-11-14T09:48:26,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T09:48:26,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 2024-11-14T09:48:26,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T09:48:26,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2b3f89fc172b9408ca73a69d8ba65061 in 1353ms, sequenceid=186, compaction requested=true 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:26,070 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:26,070 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:26,071 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:26,071 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:26,071 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:26,071 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:26,072 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:26,072 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:26,072 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1e6274136a9649f3a64c20c5a1e231b9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.9 K 2024-11-14T09:48:26,072 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=108.3 K 2024-11-14T09:48:26,072 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:26,072 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef] 2024-11-14T09:48:26,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:26,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:26,072 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fef9db6bc234ad3a26ca8898106529c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700165 2024-11-14T09:48:26,073 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6274136a9649f3a64c20c5a1e231b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700165 2024-11-14T09:48:26,073 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b7429b9e5142aeaba135aed215aa6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577702372 2024-11-14T09:48:26,073 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 70077198e23e4106bd5708453a7df29d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577702372 2024-11-14T09:48:26,073 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad76122b493b464eb2ad5aac8e9f83ef, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:26,073 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 602c8f83057445debd22bd2b11a8e950, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:26,098 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:26,102 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:26,102 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/fa9a86b7bd4d4599a48128b52391ce42 is 50, key is test_row_0/B:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:26,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T09:48:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:26,109 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:48:26,126 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ff24c645bf9b48d3911133eab9f0d921_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:26,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ff24c645bf9b48d3911133eab9f0d921_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:26,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ff24c645bf9b48d3911133eab9f0d921_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149338518b54734eedb3888488869f1407_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577704794/Put/seqid=0 2024-11-14T09:48:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742021_1197 (size=12595) 2024-11-14T09:48:26,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742022_1198 (size=4469) 2024-11-14T09:48:26,160 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#168 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:26,160 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5fb1b621de2b4a5d9bf51daa0cb90bbc is 175, key is test_row_0/A:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:26,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742023_1199 (size=12304) 2024-11-14T09:48:26,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:26,180 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149338518b54734eedb3888488869f1407_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149338518b54734eedb3888488869f1407_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:26,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ed9cf876191d462ca60a684dd8b3901c, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:26,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ed9cf876191d462ca60a684dd8b3901c is 175, key is test_row_0/A:col10/1731577704794/Put/seqid=0 2024-11-14T09:48:26,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742024_1200 (size=31549) 2024-11-14T09:48:26,192 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/5fb1b621de2b4a5d9bf51daa0cb90bbc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc 2024-11-14T09:48:26,200 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into 5fb1b621de2b4a5d9bf51daa0cb90bbc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:26,200 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:26,200 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577706070; duration=0sec 2024-11-14T09:48:26,200 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:26,200 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:26,200 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:26,202 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:26,202 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:26,202 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:26,202 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/27ab80e0fc7d455380efb6a4d7221213, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=35.9 K 2024-11-14T09:48:26,203 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27ab80e0fc7d455380efb6a4d7221213, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731577700165 2024-11-14T09:48:26,203 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90089d8e23de4c9dbadb96bb0fefd641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577702372 2024-11-14T09:48:26,203 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af9cdc0f5fa4b8aaff7ae7ceb793393, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742025_1201 (size=31105) 2024-11-14T09:48:26,214 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ed9cf876191d462ca60a684dd8b3901c 2024-11-14T09:48:26,223 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:26,224 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1b6015e3f3624b0ca3145f80db45bdf6 is 50, key is test_row_0/C:col10/1731577703532/Put/seqid=0 2024-11-14T09:48:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6f144605ce584e98b4c3c54bfd22582e is 50, key is test_row_0/B:col10/1731577704794/Put/seqid=0 2024-11-14T09:48:26,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742026_1202 (size=12595) 2024-11-14T09:48:26,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742027_1203 (size=12151) 2024-11-14T09:48:26,560 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/fa9a86b7bd4d4599a48128b52391ce42 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/fa9a86b7bd4d4599a48128b52391ce42 2024-11-14T09:48:26,571 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into fa9a86b7bd4d4599a48128b52391ce42(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:26,571 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:26,571 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577706070; duration=0sec 2024-11-14T09:48:26,571 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:26,571 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:26,640 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/1b6015e3f3624b0ca3145f80db45bdf6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1b6015e3f3624b0ca3145f80db45bdf6 2024-11-14T09:48:26,643 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6f144605ce584e98b4c3c54bfd22582e 2024-11-14T09:48:26,646 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 1b6015e3f3624b0ca3145f80db45bdf6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:26,646 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:26,646 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577706070; duration=0sec 2024-11-14T09:48:26,647 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:26,647 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:26,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/658de6165da745ef8b88c9a602beef16 is 50, key is test_row_0/C:col10/1731577704794/Put/seqid=0 2024-11-14T09:48:26,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742028_1204 (size=12151) 2024-11-14T09:48:26,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577766660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577766661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577766662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577766764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577766774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577766775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577766955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,957 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577766964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,966 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577766975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577766985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577766986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,057 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/658de6165da745ef8b88c9a602beef16 2024-11-14T09:48:27,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ed9cf876191d462ca60a684dd8b3901c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c 2024-11-14T09:48:27,066 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c, entries=150, sequenceid=208, filesize=30.4 K 2024-11-14T09:48:27,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6f144605ce584e98b4c3c54bfd22582e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e 2024-11-14T09:48:27,072 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T09:48:27,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/658de6165da745ef8b88c9a602beef16 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16 2024-11-14T09:48:27,078 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T09:48:27,079 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2b3f89fc172b9408ca73a69d8ba65061 in 970ms, sequenceid=208, compaction requested=false 2024-11-14T09:48:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T09:48:27,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T09:48:27,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T09:48:27,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3530 sec 2024-11-14T09:48:27,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.3590 sec 2024-11-14T09:48:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:27,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:27,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:48:27,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b4e5de71d4234a0d98187ef801b63bd5_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:27,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742029_1205 (size=14794) 2024-11-14T09:48:27,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577767325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577767328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577767329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577767434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577767434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577767444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577767645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577767645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577767654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:27,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b4e5de71d4234a0d98187ef801b63bd5_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b4e5de71d4234a0d98187ef801b63bd5_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:27,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/70e012b83ee14bb28cd04b8d854fa286, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:27,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/70e012b83ee14bb28cd04b8d854fa286 is 175, key is test_row_0/A:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:27,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742030_1206 (size=39749) 2024-11-14T09:48:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577767955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577767955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:27,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577767964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:27,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:28,179 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/70e012b83ee14bb28cd04b8d854fa286 2024-11-14T09:48:28,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/7ac67e5a7da643a9bb71de429ffaeccd is 50, key is test_row_0/B:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:28,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742031_1207 (size=12151) 2024-11-14T09:48:28,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577768464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:28,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577768464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:28,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577768475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:28,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/7ac67e5a7da643a9bb71de429ffaeccd 2024-11-14T09:48:28,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/86c418fa26084963bbee43255217fd72 is 50, key is test_row_0/C:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742032_1208 (size=12151) 2024-11-14T09:48:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T09:48:28,895 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:28,895 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T09:48:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:48:28,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:28,897 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:28,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:28,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577768974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:28,977 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577768985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:28,988 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:48:29,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/86c418fa26084963bbee43255217fd72 2024-11-14T09:48:29,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/70e012b83ee14bb28cd04b8d854fa286 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286 2024-11-14T09:48:29,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286, entries=200, sequenceid=226, filesize=38.8 K 2024-11-14T09:48:29,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/7ac67e5a7da643a9bb71de429ffaeccd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd 2024-11-14T09:48:29,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T09:48:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/86c418fa26084963bbee43255217fd72 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72 2024-11-14T09:48:29,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72, entries=150, sequenceid=226, filesize=11.9 K 2024-11-14T09:48:29,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2b3f89fc172b9408ca73a69d8ba65061 in 1763ms, sequenceid=226, compaction requested=true 2024-11-14T09:48:29,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:29,050 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:29,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:29,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:29,051 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:29,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T09:48:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:29,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,051 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:48:29,054 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:29,054 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:29,054 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,054 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=100.0 K 2024-11-14T09:48:29,054 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,054 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286] 2024-11-14T09:48:29,054 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:29,054 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:29,054 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,054 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/fa9a86b7bd4d4599a48128b52391ce42, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.0 K 2024-11-14T09:48:29,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb1b621de2b4a5d9bf51daa0cb90bbc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:29,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed9cf876191d462ca60a684dd8b3901c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731577704794 2024-11-14T09:48:29,055 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9a86b7bd4d4599a48128b52391ce42, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:29,056 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e012b83ee14bb28cd04b8d854fa286, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:29,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f144605ce584e98b4c3c54bfd22582e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731577704794 2024-11-14T09:48:29,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac67e5a7da643a9bb71de429ffaeccd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:29,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147ecf123724e749be976f5ddc7b16b820_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577707323/Put/seqid=0 2024-11-14T09:48:29,077 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:29,078 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#179 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:29,078 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6eb0f2c342454f9794d941baacd4b8f0 is 50, key is test_row_0/B:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:29,085 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411144a380ddf5e704f2b8f568f7ce92d7543_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:29,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742033_1209 (size=12304) 2024-11-14T09:48:29,087 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411144a380ddf5e704f2b8f568f7ce92d7543_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:29,087 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144a380ddf5e704f2b8f568f7ce92d7543_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:29,095 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147ecf123724e749be976f5ddc7b16b820_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147ecf123724e749be976f5ddc7b16b820_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:29,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/d6dc46824d6a4f64b585cdddf626af0f, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:29,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/d6dc46824d6a4f64b585cdddf626af0f is 175, key is test_row_0/A:col10/1731577707323/Put/seqid=0 2024-11-14T09:48:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742035_1211 (size=4469) 2024-11-14T09:48:29,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742034_1210 (size=12697) 2024-11-14T09:48:29,110 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6eb0f2c342454f9794d941baacd4b8f0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6eb0f2c342454f9794d941baacd4b8f0 2024-11-14T09:48:29,116 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into 6eb0f2c342454f9794d941baacd4b8f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:29,116 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:29,117 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577709051; duration=0sec 2024-11-14T09:48:29,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:29,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:29,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:29,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:29,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:29,118 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,118 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1b6015e3f3624b0ca3145f80db45bdf6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.0 K 2024-11-14T09:48:29,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b6015e3f3624b0ca3145f80db45bdf6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731577703532 2024-11-14T09:48:29,119 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 658de6165da745ef8b88c9a602beef16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731577704794 2024-11-14T09:48:29,120 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c418fa26084963bbee43255217fd72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:29,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742036_1212 (size=31105) 2024-11-14T09:48:29,131 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:29,132 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/6b11cf5639444782994283022e5bf2ac is 50, key is test_row_0/C:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:29,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742037_1213 (size=12697) 2024-11-14T09:48:29,159 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/6b11cf5639444782994283022e5bf2ac as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6b11cf5639444782994283022e5bf2ac 2024-11-14T09:48:29,166 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 6b11cf5639444782994283022e5bf2ac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:29,166 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:29,166 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577709051; duration=0sec 2024-11-14T09:48:29,166 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:29,166 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:48:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:29,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:29,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577769491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577769493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577769494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,500 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:29,500 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/1ab387d7a12d49fd8a64c9bfb462f5f4 is 175, key is test_row_0/A:col10/1731577706657/Put/seqid=0 2024-11-14T09:48:29,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742038_1214 (size=31651) 2024-11-14T09:48:29,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/1ab387d7a12d49fd8a64c9bfb462f5f4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4 2024-11-14T09:48:29,513 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into 1ab387d7a12d49fd8a64c9bfb462f5f4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:29,513 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:29,513 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577709050; duration=0sec 2024-11-14T09:48:29,513 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:29,513 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:29,524 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/d6dc46824d6a4f64b585cdddf626af0f 2024-11-14T09:48:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:48:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/beb8ef5b6f604f32845a337796a5c9c0 is 50, key is test_row_0/B:col10/1731577707323/Put/seqid=0 2024-11-14T09:48:29,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742039_1215 (size=12151) 2024-11-14T09:48:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577769604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577769605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577769605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577769814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577769814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577769814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:29,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:29,937 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/beb8ef5b6f604f32845a337796a5c9c0 2024-11-14T09:48:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8a2defd0c6f64404a0e2b32f8eaedc9d is 50, key is test_row_0/C:col10/1731577707323/Put/seqid=0 2024-11-14T09:48:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742040_1216 (size=12151) 2024-11-14T09:48:29,949 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8a2defd0c6f64404a0e2b32f8eaedc9d 2024-11-14T09:48:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/d6dc46824d6a4f64b585cdddf626af0f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f 2024-11-14T09:48:29,957 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f, entries=150, sequenceid=247, filesize=30.4 K 2024-11-14T09:48:29,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/beb8ef5b6f604f32845a337796a5c9c0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0 2024-11-14T09:48:29,963 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T09:48:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8a2defd0c6f64404a0e2b32f8eaedc9d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d 2024-11-14T09:48:29,968 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T09:48:29,969 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2b3f89fc172b9408ca73a69d8ba65061 in 918ms, sequenceid=247, compaction requested=false 2024-11-14T09:48:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T09:48:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T09:48:29,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T09:48:29,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0730 sec 2024-11-14T09:48:29,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.0770 sec 2024-11-14T09:48:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T09:48:30,034 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:30,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T09:48:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:30,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:30,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:30,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:30,128 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:30,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:48:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:30,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb68050ec4c94889a8ce8a7de0871ed8_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:30,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742041_1217 (size=12454) 2024-11-14T09:48:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577770165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577770166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577770167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577770274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577770275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577770275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:30,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577770485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577770485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577770485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,550 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:30,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb68050ec4c94889a8ce8a7de0871ed8_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb68050ec4c94889a8ce8a7de0871ed8_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:30,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/24455d9990e2443788522ef8b0fc4534, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:30,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/24455d9990e2443788522ef8b0fc4534 is 175, key is test_row_0/A:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:30,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742042_1218 (size=31255) 2024-11-14T09:48:30,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/24455d9990e2443788522ef8b0fc4534 2024-11-14T09:48:30,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/f6622bc9aa2f4968b509439ea8331d9a is 50, key is test_row_0/B:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:30,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742043_1219 (size=12301) 2024-11-14T09:48:30,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577770795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577770795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577770795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:30,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:30,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:30,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:30,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/f6622bc9aa2f4968b509439ea8331d9a 2024-11-14T09:48:31,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/7bef38bebead4fecb0842884dc227659 is 50, key is test_row_0/C:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:31,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742044_1220 (size=12301) 2024-11-14T09:48:31,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:31,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:31,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:31,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:31,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577771305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:31,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577771315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:31,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577771315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:31,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:31,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/7bef38bebead4fecb0842884dc227659 2024-11-14T09:48:31,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/24455d9990e2443788522ef8b0fc4534 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534 2024-11-14T09:48:31,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:31,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534, entries=150, sequenceid=268, filesize=30.5 K 2024-11-14T09:48:31,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/f6622bc9aa2f4968b509439ea8331d9a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a 2024-11-14T09:48:31,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a, entries=150, sequenceid=268, filesize=12.0 K 2024-11-14T09:48:31,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/7bef38bebead4fecb0842884dc227659 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659 2024-11-14T09:48:31,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659, entries=150, sequenceid=268, filesize=12.0 K 2024-11-14T09:48:31,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2b3f89fc172b9408ca73a69d8ba65061 in 1310ms, sequenceid=268, compaction requested=true 2024-11-14T09:48:31,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:31,438 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:31,440 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:31,440 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:31,440 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,440 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=91.8 K 2024-11-14T09:48:31,440 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,440 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534] 2024-11-14T09:48:31,441 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab387d7a12d49fd8a64c9bfb462f5f4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:31,441 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6dc46824d6a4f64b585cdddf626af0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577707323 2024-11-14T09:48:31,441 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24455d9990e2443788522ef8b0fc4534, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:31,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:31,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:31,442 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:31,444 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:31,444 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:31,444 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,444 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6eb0f2c342454f9794d941baacd4b8f0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.3 K 2024-11-14T09:48:31,445 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb0f2c342454f9794d941baacd4b8f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:31,447 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting beb8ef5b6f604f32845a337796a5c9c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577707323 2024-11-14T09:48:31,447 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f6622bc9aa2f4968b509439ea8331d9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:31,452 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:31,458 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:31,459 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/0cd51a79ad80448ea9f008b3f4172d77 is 50, key is test_row_0/B:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:31,463 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a0f150aea8284935a92be523aafca49f_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:31,466 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a0f150aea8284935a92be523aafca49f_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:31,466 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a0f150aea8284935a92be523aafca49f_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:31,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742045_1221 (size=12949) 2024-11-14T09:48:31,484 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/0cd51a79ad80448ea9f008b3f4172d77 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/0cd51a79ad80448ea9f008b3f4172d77 2024-11-14T09:48:31,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742046_1222 (size=4469) 2024-11-14T09:48:31,490 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into 0cd51a79ad80448ea9f008b3f4172d77(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:31,490 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:31,490 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577711441; duration=0sec 2024-11-14T09:48:31,491 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:31,491 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:31,491 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:31,492 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:31,492 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:31,492 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,493 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6b11cf5639444782994283022e5bf2ac, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.3 K 2024-11-14T09:48:31,493 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b11cf5639444782994283022e5bf2ac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731577706657 2024-11-14T09:48:31,494 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2defd0c6f64404a0e2b32f8eaedc9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577707323 2024-11-14T09:48:31,494 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bef38bebead4fecb0842884dc227659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:31,498 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#186 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:31,499 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/e51051ce450d4c3f82757a1e663e5cab is 175, key is test_row_0/A:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:31,506 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:31,506 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/3c4cdd6349ca4d98b58635e12163193e is 50, key is test_row_0/C:col10/1731577709488/Put/seqid=0 2024-11-14T09:48:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742047_1223 (size=12949) 2024-11-14T09:48:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742048_1224 (size=31903) 2024-11-14T09:48:31,542 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/e51051ce450d4c3f82757a1e663e5cab as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab 2024-11-14T09:48:31,548 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into e51051ce450d4c3f82757a1e663e5cab(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:31,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:31,548 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577711438; duration=0sec 2024-11-14T09:48:31,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:31,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:31,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T09:48:31,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:31,569 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T09:48:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111408159e60f80f4febb4380c45ad338545_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577710165/Put/seqid=0 2024-11-14T09:48:31,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742049_1225 (size=12454) 2024-11-14T09:48:31,923 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/3c4cdd6349ca4d98b58635e12163193e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/3c4cdd6349ca4d98b58635e12163193e 2024-11-14T09:48:31,927 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 3c4cdd6349ca4d98b58635e12163193e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:31,927 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:31,927 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577711442; duration=0sec 2024-11-14T09:48:31,927 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:31,927 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:31,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:31,986 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111408159e60f80f4febb4380c45ad338545_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111408159e60f80f4febb4380c45ad338545_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:31,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/4c933c7770c544c29c58d6d0bc597f6b, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:31,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/4c933c7770c544c29c58d6d0bc597f6b is 175, key is test_row_0/A:col10/1731577710165/Put/seqid=0 2024-11-14T09:48:31,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742050_1226 (size=31255) 2024-11-14T09:48:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:32,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577772338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577772339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577772340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,394 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/4c933c7770c544c29c58d6d0bc597f6b 2024-11-14T09:48:32,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/e3a734fb573c49a1b5a162fc6ecd0d1a is 50, key is test_row_0/B:col10/1731577710165/Put/seqid=0 2024-11-14T09:48:32,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742051_1227 (size=12301) 2024-11-14T09:48:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/e3a734fb573c49a1b5a162fc6ecd0d1a 2024-11-14T09:48:32,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8087eb540b7d49f08bc1b6a6c049e445 is 50, key is test_row_0/C:col10/1731577710165/Put/seqid=0 2024-11-14T09:48:32,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742052_1228 (size=12301) 2024-11-14T09:48:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577772444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577772444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577772445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577772655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577772655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577772655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,829 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8087eb540b7d49f08bc1b6a6c049e445 2024-11-14T09:48:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/4c933c7770c544c29c58d6d0bc597f6b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b 2024-11-14T09:48:32,915 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b, entries=150, sequenceid=288, filesize=30.5 K 2024-11-14T09:48:32,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/e3a734fb573c49a1b5a162fc6ecd0d1a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a 2024-11-14T09:48:32,919 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T09:48:32,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/8087eb540b7d49f08bc1b6a6c049e445 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445 2024-11-14T09:48:32,925 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T09:48:32,926 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 2b3f89fc172b9408ca73a69d8ba65061 in 1357ms, sequenceid=288, compaction requested=false 2024-11-14T09:48:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T09:48:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T09:48:32,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T09:48:32,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8900 sec 2024-11-14T09:48:32,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.8940 sec 2024-11-14T09:48:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:32,967 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:32,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T09:48:32,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111414a0549accde4f529bdb1c0e83dd8b07_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742053_1229 (size=17534) 2024-11-14T09:48:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577772982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577772984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577772985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37332 deadline: 1731577773014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,016 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37392 deadline: 1731577773024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,026 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 2b3f89fc172b9408ca73a69d8ba65061 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:33,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577773095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577773095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577773095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577773304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577773304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577773305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:33,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111414a0549accde4f529bdb1c0e83dd8b07_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111414a0549accde4f529bdb1c0e83dd8b07_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:33,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/05aebb5bad134c65ad9b12168dc35763, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:33,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/05aebb5bad134c65ad9b12168dc35763 is 175, key is test_row_0/A:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:33,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742054_1230 (size=48639) 2024-11-14T09:48:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577773615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577773615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577773615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:33,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:33,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/05aebb5bad134c65ad9b12168dc35763 2024-11-14T09:48:33,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6ef44ac00ed647829d3dd2801ca52ec6 is 50, key is test_row_0/B:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:33,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742055_1231 (size=12301) 2024-11-14T09:48:33,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6ef44ac00ed647829d3dd2801ca52ec6 2024-11-14T09:48:33,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a1df3ee5dabd4f1ba914431ffe3d3980 is 50, key is test_row_0/C:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:33,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742056_1232 (size=12301) 2024-11-14T09:48:34,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37372 deadline: 1731577774125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:34,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37388 deadline: 1731577774125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37324 deadline: 1731577774125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:48:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T09:48:34,205 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:34,206 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T09:48:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:34,207 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:34,208 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:34,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:34,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a1df3ee5dabd4f1ba914431ffe3d3980 2024-11-14T09:48:34,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/05aebb5bad134c65ad9b12168dc35763 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763 2024-11-14T09:48:34,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763, entries=250, sequenceid=309, filesize=47.5 K 2024-11-14T09:48:34,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/6ef44ac00ed647829d3dd2801ca52ec6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6 2024-11-14T09:48:34,233 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-14T09:48:34,233 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:34,233 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:34,234 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,237 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-14T09:48:34,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6, entries=150, sequenceid=309, filesize=12.0 K 2024-11-14T09:48:34,237 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:34,237 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,237 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:34,238 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-14T09:48:34,238 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:34,238 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,238 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:34,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/a1df3ee5dabd4f1ba914431ffe3d3980 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980 2024-11-14T09:48:34,239 INFO [Thread-724 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-724. 2024-11-14T09:48:34,239 DEBUG [Thread-724 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:34,239 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,239 DEBUG [Thread-724 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:34,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:34,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980, entries=150, sequenceid=309, filesize=12.0 K 2024-11-14T09:48:34,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2b3f89fc172b9408ca73a69d8ba65061 in 1275ms, sequenceid=309, compaction requested=true 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:34,243 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b3f89fc172b9408ca73a69d8ba65061:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:34,243 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:34,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/B is initiating minor compaction (all files) 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/A is initiating minor compaction (all files) 2024-11-14T09:48:34,244 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/A in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:34,244 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/B in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:34,244 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=109.2 K 2024-11-14T09:48:34,244 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/0cd51a79ad80448ea9f008b3f4172d77, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.7 K 2024-11-14T09:48:34,244 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763] 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cd51a79ad80448ea9f008b3f4172d77, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:34,244 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e51051ce450d4c3f82757a1e663e5cab, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:34,245 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a734fb573c49a1b5a162fc6ecd0d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577710158 2024-11-14T09:48:34,245 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c933c7770c544c29c58d6d0bc597f6b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577710158 2024-11-14T09:48:34,245 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ef44ac00ed647829d3dd2801ca52ec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731577712338 2024-11-14T09:48:34,245 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05aebb5bad134c65ad9b12168dc35763, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731577712331 2024-11-14T09:48:34,250 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:34,251 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:34,251 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e9bd781bebf24bf19c58a14fddda113f_2b3f89fc172b9408ca73a69d8ba65061 store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:34,251 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/df6b3c60a4894f18a742a9169016cf15 is 50, key is test_row_0/B:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:34,253 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e9bd781bebf24bf19c58a14fddda113f_2b3f89fc172b9408ca73a69d8ba65061, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:34,253 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e9bd781bebf24bf19c58a14fddda113f_2b3f89fc172b9408ca73a69d8ba65061 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:34,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742057_1233 (size=13051) 2024-11-14T09:48:34,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742058_1234 (size=4469) 2024-11-14T09:48:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:34,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T09:48:34,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:34,360 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:48:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a26a7e37970d4d37b9725fb10d1caa49_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577712983/Put/seqid=0 2024-11-14T09:48:34,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742059_1235 (size=12454) 2024-11-14T09:48:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:34,662 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#A#compaction#195 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:34,663 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ba1e60a1048b4870bdca77b6ce4e2562 is 175, key is test_row_0/A:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:34,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742060_1236 (size=32005) 2024-11-14T09:48:34,671 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/df6b3c60a4894f18a742a9169016cf15 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/df6b3c60a4894f18a742a9169016cf15 2024-11-14T09:48:34,676 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/B of 2b3f89fc172b9408ca73a69d8ba65061 into df6b3c60a4894f18a742a9169016cf15(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:34,676 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:34,676 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/B, priority=13, startTime=1731577714243; duration=0sec 2024-11-14T09:48:34,676 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:34,676 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:B 2024-11-14T09:48:34,676 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:34,677 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:34,677 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 2b3f89fc172b9408ca73a69d8ba65061/C is initiating minor compaction (all files) 2024-11-14T09:48:34,677 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b3f89fc172b9408ca73a69d8ba65061/C in TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:34,677 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/3c4cdd6349ca4d98b58635e12163193e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp, totalSize=36.7 K 2024-11-14T09:48:34,678 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c4cdd6349ca4d98b58635e12163193e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731577709488 2024-11-14T09:48:34,678 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8087eb540b7d49f08bc1b6a6c049e445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577710158 2024-11-14T09:48:34,678 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a1df3ee5dabd4f1ba914431ffe3d3980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731577712338 2024-11-14T09:48:34,686 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b3f89fc172b9408ca73a69d8ba65061#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:34,686 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/6cc17f0b240240c29abade03ddb90ac8 is 50, key is test_row_0/C:col10/1731577712966/Put/seqid=0 2024-11-14T09:48:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742061_1237 (size=13051) 2024-11-14T09:48:34,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:34,785 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a26a7e37970d4d37b9725fb10d1caa49_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a26a7e37970d4d37b9725fb10d1caa49_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:34,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/693930655f0d4c688ff65335d3625ce8, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:34,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/693930655f0d4c688ff65335d3625ce8 is 175, key is test_row_0/A:col10/1731577712983/Put/seqid=0 2024-11-14T09:48:34,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742062_1238 (size=31255) 2024-11-14T09:48:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:35,083 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/ba1e60a1048b4870bdca77b6ce4e2562 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ba1e60a1048b4870bdca77b6ce4e2562 2024-11-14T09:48:35,090 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/A of 2b3f89fc172b9408ca73a69d8ba65061 into ba1e60a1048b4870bdca77b6ce4e2562(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:35,090 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:35,090 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/A, priority=13, startTime=1731577714243; duration=0sec 2024-11-14T09:48:35,090 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:35,090 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:A 2024-11-14T09:48:35,096 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/6cc17f0b240240c29abade03ddb90ac8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6cc17f0b240240c29abade03ddb90ac8 2024-11-14T09:48:35,102 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b3f89fc172b9408ca73a69d8ba65061/C of 2b3f89fc172b9408ca73a69d8ba65061 into 6cc17f0b240240c29abade03ddb90ac8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:35,102 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:35,102 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., storeName=2b3f89fc172b9408ca73a69d8ba65061/C, priority=13, startTime=1731577714243; duration=0sec 2024-11-14T09:48:35,103 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:35,103 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b3f89fc172b9408ca73a69d8ba65061:C 2024-11-14T09:48:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:35,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. as already flushing 2024-11-14T09:48:35,138 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-14T09:48:35,138 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-14T09:48:35,138 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:35,138 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:35,138 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,138 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,138 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,138 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:35,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:35,147 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-14T09:48:35,147 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:35,147 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,147 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:35,147 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/693930655f0d4c688ff65335d3625ce8 2024-11-14T09:48:35,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b3b7b3882cf341bd8ea0f41aa14084dc is 50, key is test_row_0/B:col10/1731577712983/Put/seqid=0 2024-11-14T09:48:35,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742063_1239 (size=12301) 2024-11-14T09:48:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:35,612 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b3b7b3882cf341bd8ea0f41aa14084dc 2024-11-14T09:48:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/071b150e5ea1433495a2e6c1bfde55e1 is 50, key is test_row_0/C:col10/1731577712983/Put/seqid=0 2024-11-14T09:48:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742064_1240 (size=12301) 2024-11-14T09:48:36,022 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/071b150e5ea1433495a2e6c1bfde55e1 2024-11-14T09:48:36,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/693930655f0d4c688ff65335d3625ce8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/693930655f0d4c688ff65335d3625ce8 2024-11-14T09:48:36,031 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/693930655f0d4c688ff65335d3625ce8, entries=150, sequenceid=325, filesize=30.5 K 2024-11-14T09:48:36,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/b3b7b3882cf341bd8ea0f41aa14084dc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b3b7b3882cf341bd8ea0f41aa14084dc 2024-11-14T09:48:36,035 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b3b7b3882cf341bd8ea0f41aa14084dc, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T09:48:36,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/071b150e5ea1433495a2e6c1bfde55e1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/071b150e5ea1433495a2e6c1bfde55e1 2024-11-14T09:48:36,040 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/071b150e5ea1433495a2e6c1bfde55e1, entries=150, sequenceid=325, filesize=12.0 K 2024-11-14T09:48:36,041 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=20.13 KB/20610 for 2b3f89fc172b9408ca73a69d8ba65061 in 1681ms, sequenceid=325, compaction requested=false 2024-11-14T09:48:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 2b3f89fc172b9408ca73a69d8ba65061: 2024-11-14T09:48:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:36,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T09:48:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T09:48:36,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T09:48:36,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-14T09:48:36,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8380 sec 2024-11-14T09:48:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T09:48:36,354 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:36,376 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:48:43,050 INFO [Thread-715 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-715. 2024-11-14T09:48:43,050 DEBUG [Thread-715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:43,051 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,051 DEBUG [Thread-715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:43,087 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-14T09:48:43,087 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:48:43,087 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,087 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4036 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3985 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1775 2024-11-14T09:48:43,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5324 rows 2024-11-14T09:48:43,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1764 2024-11-14T09:48:43,088 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5289 rows 2024-11-14T09:48:43,088 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:48:43,088 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:48:43,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:43,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:48:43,089 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:48:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:43,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577723092"}]},"ts":"1731577723092"} 2024-11-14T09:48:43,094 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:48:43,094 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:48:43,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:48:43,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, UNASSIGN}] 2024-11-14T09:48:43,098 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, UNASSIGN 2024-11-14T09:48:43,099 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:43,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, UNASSIGN because future has completed 2024-11-14T09:48:43,101 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:48:43,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:43,254 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:43,254 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:48:43,255 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 2b3f89fc172b9408ca73a69d8ba65061, disabling compactions & flushes 2024-11-14T09:48:43,255 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:43,255 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:43,255 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. after waiting 0 ms 2024-11-14T09:48:43,255 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:43,255 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 2b3f89fc172b9408ca73a69d8ba65061 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T09:48:43,268 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141ff3b5c36e874a7f89b283d5e1d71c66_2b3f89fc172b9408ca73a69d8ba65061 is 50, key is test_row_0/A:col10/1731577715145/Put/seqid=0 2024-11-14T09:48:43,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742065_1241 (size=12454) 2024-11-14T09:48:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:43,674 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:43,684 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141ff3b5c36e874a7f89b283d5e1d71c66_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ff3b5c36e874a7f89b283d5e1d71c66_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:43,686 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/b4d38114c25b4f4ba1b71783ced7266c, store: [table=TestAcidGuarantees family=A region=2b3f89fc172b9408ca73a69d8ba65061] 2024-11-14T09:48:43,686 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/b4d38114c25b4f4ba1b71783ced7266c is 175, key is test_row_0/A:col10/1731577715145/Put/seqid=0 2024-11-14T09:48:43,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742066_1242 (size=31255) 2024-11-14T09:48:43,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:44,091 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/b4d38114c25b4f4ba1b71783ced7266c 2024-11-14T09:48:44,105 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4b032bdb536e4ab2833e37cde77bcde5 is 50, key is test_row_0/B:col10/1731577715145/Put/seqid=0 2024-11-14T09:48:44,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742067_1243 (size=12301) 2024-11-14T09:48:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:44,512 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4b032bdb536e4ab2833e37cde77bcde5 2024-11-14T09:48:44,525 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/f18a3d1b3fdc42febe959be7cfb5acd0 is 50, key is test_row_0/C:col10/1731577715145/Put/seqid=0 2024-11-14T09:48:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742068_1244 (size=12301) 2024-11-14T09:48:44,931 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/f18a3d1b3fdc42febe959be7cfb5acd0 2024-11-14T09:48:44,941 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/A/b4d38114c25b4f4ba1b71783ced7266c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/b4d38114c25b4f4ba1b71783ced7266c 2024-11-14T09:48:44,948 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/b4d38114c25b4f4ba1b71783ced7266c, entries=150, sequenceid=336, filesize=30.5 K 2024-11-14T09:48:44,949 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/B/4b032bdb536e4ab2833e37cde77bcde5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4b032bdb536e4ab2833e37cde77bcde5 2024-11-14T09:48:44,953 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4b032bdb536e4ab2833e37cde77bcde5, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T09:48:44,954 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/.tmp/C/f18a3d1b3fdc42febe959be7cfb5acd0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/f18a3d1b3fdc42febe959be7cfb5acd0 2024-11-14T09:48:44,959 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/f18a3d1b3fdc42febe959be7cfb5acd0, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T09:48:44,960 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2b3f89fc172b9408ca73a69d8ba65061 in 1705ms, sequenceid=336, compaction requested=true 2024-11-14T09:48:44,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763] to archive 2024-11-14T09:48:44,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:44,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/0b90be113db84cd589fce9cf159d34c3 2024-11-14T09:48:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/f0065c541efb41fa859b7b3d64df0bd0 2024-11-14T09:48:44,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/2511e673946648ee83c71670a410c5ff 2024-11-14T09:48:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/37cd26fbcb6c4c3bb469a918cf4a4606 2024-11-14T09:48:44,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/c20111374b3442dd970a704f8bbfbd93 2024-11-14T09:48:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5aad033fd2524318824cf58f2f98ca0e 2024-11-14T09:48:44,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/80a006bb51494bf88b663188997f1b71 2024-11-14T09:48:44,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/de86ab0476744378b800477bf97d69cb 2024-11-14T09:48:44,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/9fea2f6765c949449f3485b061f83b83 2024-11-14T09:48:44,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/fdbea4a1d20c46f19f8a22b61c8e6ff8 2024-11-14T09:48:44,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/7fef9db6bc234ad3a26ca8898106529c 2024-11-14T09:48:44,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/22b7429b9e5142aeaba135aed215aa6d 2024-11-14T09:48:44,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ad76122b493b464eb2ad5aac8e9f83ef 2024-11-14T09:48:44,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/5fb1b621de2b4a5d9bf51daa0cb90bbc 2024-11-14T09:48:44,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ed9cf876191d462ca60a684dd8b3901c 2024-11-14T09:48:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/70e012b83ee14bb28cd04b8d854fa286 2024-11-14T09:48:44,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/1ab387d7a12d49fd8a64c9bfb462f5f4 2024-11-14T09:48:44,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/d6dc46824d6a4f64b585cdddf626af0f 2024-11-14T09:48:44,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/e51051ce450d4c3f82757a1e663e5cab 2024-11-14T09:48:44,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/24455d9990e2443788522ef8b0fc4534 2024-11-14T09:48:44,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/4c933c7770c544c29c58d6d0bc597f6b 2024-11-14T09:48:44,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/05aebb5bad134c65ad9b12168dc35763 2024-11-14T09:48:44,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/81529dce73014c569f382df14e746f1e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/2758b86e2baa47918c273cf3e5a66b0b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1e6274136a9649f3a64c20c5a1e231b9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/fa9a86b7bd4d4599a48128b52391ce42, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6eb0f2c342454f9794d941baacd4b8f0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/0cd51a79ad80448ea9f008b3f4172d77, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6] to archive 2024-11-14T09:48:44,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:44,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/728dd5e467494befbcf2be5e83b78ff8 2024-11-14T09:48:44,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/732ba0ef15f04faaa96a6da64d517f65 2024-11-14T09:48:44,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/00e68a52a5994ba8ab41266372538b51 2024-11-14T09:48:44,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/81529dce73014c569f382df14e746f1e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/81529dce73014c569f382df14e746f1e 2024-11-14T09:48:44,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/c6ea5a27ae0c46de93857fe0f2177837 2024-11-14T09:48:44,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4af83460973e42de885168fd696b3d2f 2024-11-14T09:48:44,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/2758b86e2baa47918c273cf3e5a66b0b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/2758b86e2baa47918c273cf3e5a66b0b 2024-11-14T09:48:44,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1f6144c1b7554a4ea17914ab643eca87 2024-11-14T09:48:44,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/8e305e5197a14d9b8e969ac5b5b9060b 2024-11-14T09:48:45,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1e6274136a9649f3a64c20c5a1e231b9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/1e6274136a9649f3a64c20c5a1e231b9 2024-11-14T09:48:45,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b05a6c8b8b1a4f209a3b90c2e43eef4e 2024-11-14T09:48:45,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/70077198e23e4106bd5708453a7df29d 2024-11-14T09:48:45,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/fa9a86b7bd4d4599a48128b52391ce42 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/fa9a86b7bd4d4599a48128b52391ce42 2024-11-14T09:48:45,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/602c8f83057445debd22bd2b11a8e950 2024-11-14T09:48:45,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6f144605ce584e98b4c3c54bfd22582e 2024-11-14T09:48:45,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6eb0f2c342454f9794d941baacd4b8f0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6eb0f2c342454f9794d941baacd4b8f0 2024-11-14T09:48:45,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/7ac67e5a7da643a9bb71de429ffaeccd 2024-11-14T09:48:45,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/beb8ef5b6f604f32845a337796a5c9c0 2024-11-14T09:48:45,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/0cd51a79ad80448ea9f008b3f4172d77 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/0cd51a79ad80448ea9f008b3f4172d77 2024-11-14T09:48:45,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/f6622bc9aa2f4968b509439ea8331d9a 2024-11-14T09:48:45,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/e3a734fb573c49a1b5a162fc6ecd0d1a 2024-11-14T09:48:45,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/6ef44ac00ed647829d3dd2801ca52ec6 2024-11-14T09:48:45,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/0cba974a549d442ca3f1ca31d007f639, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/e704e9d12daa4b478d3f4c29c4e3cfb0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/27ab80e0fc7d455380efb6a4d7221213, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1b6015e3f3624b0ca3145f80db45bdf6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6b11cf5639444782994283022e5bf2ac, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/3c4cdd6349ca4d98b58635e12163193e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980] to archive 2024-11-14T09:48:45,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:48:45,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8f5f45f80c3f4a518a36cb92e31576ed 2024-11-14T09:48:45,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1f9af4e2e73143d5804146e715d2c72b 2024-11-14T09:48:45,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1587b5250f7642a4a5b50e0b6c6474d6 2024-11-14T09:48:45,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/0cba974a549d442ca3f1ca31d007f639 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/0cba974a549d442ca3f1ca31d007f639 2024-11-14T09:48:45,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/37d17bc98ad24114ba1989f457a7d139 2024-11-14T09:48:45,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/21297aa82b0b43a4b4aa7b155b986e51 2024-11-14T09:48:45,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/e704e9d12daa4b478d3f4c29c4e3cfb0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/e704e9d12daa4b478d3f4c29c4e3cfb0 2024-11-14T09:48:45,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a8c518edcb1b4630bc38ae270da31a1f 2024-11-14T09:48:45,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/c74391fd3c0a466180dea0d6abbef124 2024-11-14T09:48:45,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/27ab80e0fc7d455380efb6a4d7221213 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/27ab80e0fc7d455380efb6a4d7221213 2024-11-14T09:48:45,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/14cbd3ace9d94f28b4cce7e9b0995736 2024-11-14T09:48:45,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/90089d8e23de4c9dbadb96bb0fefd641 2024-11-14T09:48:45,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1b6015e3f3624b0ca3145f80db45bdf6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/1b6015e3f3624b0ca3145f80db45bdf6 2024-11-14T09:48:45,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8af9cdc0f5fa4b8aaff7ae7ceb793393 2024-11-14T09:48:45,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/658de6165da745ef8b88c9a602beef16 2024-11-14T09:48:45,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6b11cf5639444782994283022e5bf2ac to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6b11cf5639444782994283022e5bf2ac 2024-11-14T09:48:45,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/86c418fa26084963bbee43255217fd72 2024-11-14T09:48:45,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8a2defd0c6f64404a0e2b32f8eaedc9d 2024-11-14T09:48:45,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/3c4cdd6349ca4d98b58635e12163193e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/3c4cdd6349ca4d98b58635e12163193e 2024-11-14T09:48:45,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/7bef38bebead4fecb0842884dc227659 2024-11-14T09:48:45,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/8087eb540b7d49f08bc1b6a6c049e445 2024-11-14T09:48:45,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/a1df3ee5dabd4f1ba914431ffe3d3980 2024-11-14T09:48:45,039 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits/339.seqid, newMaxSeqId=339, maxSeqId=4 2024-11-14T09:48:45,040 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. 2024-11-14T09:48:45,040 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 2b3f89fc172b9408ca73a69d8ba65061: Waiting for close lock at 1731577723254Running coprocessor pre-close hooks at 1731577723254Disabling compacts and flushes for region at 1731577723255 (+1 ms)Disabling writes for close at 1731577723255Obtaining lock to block concurrent updates at 1731577723255Preparing flush snapshotting stores in 2b3f89fc172b9408ca73a69d8ba65061 at 1731577723255Finished memstore snapshotting TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731577723256 (+1 ms)Flushing stores of TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061. at 1731577723258 (+2 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/A: creating writer at 1731577723258Flushing mob file 2b3f89fc172b9408ca73a69d8ba65061/A: appending metadata at 1731577723268 (+10 ms)Flushing mob file 2b3f89fc172b9408ca73a69d8ba65061/A: closing flushed file at 1731577723268Flushing 2b3f89fc172b9408ca73a69d8ba65061/A: appending metadata at 1731577723686 (+418 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/A: closing flushed file at 1731577723686Flushing 2b3f89fc172b9408ca73a69d8ba65061/B: creating writer at 1731577724100 (+414 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/B: appending metadata at 1731577724104 (+4 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/B: closing flushed file at 1731577724104Flushing 2b3f89fc172b9408ca73a69d8ba65061/C: creating writer at 1731577724522 (+418 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/C: appending metadata at 1731577724525 (+3 ms)Flushing 2b3f89fc172b9408ca73a69d8ba65061/C: closing flushed file at 1731577724525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3264086f: reopening flushed file at 1731577724939 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f6557c: reopening flushed file at 1731577724948 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@718c27dd: reopening flushed file at 1731577724954 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2b3f89fc172b9408ca73a69d8ba65061 in 1705ms, sequenceid=336, compaction requested=true at 1731577724960 (+6 ms)Writing region close event to WAL at 1731577725036 (+76 ms)Running coprocessor post-close hooks at 1731577725040 (+4 ms)Closed at 1731577725040 2024-11-14T09:48:45,042 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,042 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=2b3f89fc172b9408ca73a69d8ba65061, regionState=CLOSED 2024-11-14T09:48:45,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:45,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T09:48:45,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 2b3f89fc172b9408ca73a69d8ba65061, server=defc576eb6b7,40243,1731577658084 in 1.9430 sec 2024-11-14T09:48:45,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-14T09:48:45,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b3f89fc172b9408ca73a69d8ba65061, UNASSIGN in 1.9480 sec 2024-11-14T09:48:45,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T09:48:45,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-14T09:48:45,049 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577725049"}]},"ts":"1731577725049"} 2024-11-14T09:48:45,050 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:48:45,050 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:48:45,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9610 sec 2024-11-14T09:48:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T09:48:45,235 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:45,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:48:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:48:45,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,243 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits] 2024-11-14T09:48:45,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/693930655f0d4c688ff65335d3625ce8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/693930655f0d4c688ff65335d3625ce8 2024-11-14T09:48:45,251 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/b4d38114c25b4f4ba1b71783ced7266c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/b4d38114c25b4f4ba1b71783ced7266c 2024-11-14T09:48:45,252 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ba1e60a1048b4870bdca77b6ce4e2562 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/A/ba1e60a1048b4870bdca77b6ce4e2562 2024-11-14T09:48:45,254 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4b032bdb536e4ab2833e37cde77bcde5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/4b032bdb536e4ab2833e37cde77bcde5 2024-11-14T09:48:45,255 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b3b7b3882cf341bd8ea0f41aa14084dc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/b3b7b3882cf341bd8ea0f41aa14084dc 2024-11-14T09:48:45,256 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/df6b3c60a4894f18a742a9169016cf15 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/B/df6b3c60a4894f18a742a9169016cf15 2024-11-14T09:48:45,258 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/071b150e5ea1433495a2e6c1bfde55e1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/071b150e5ea1433495a2e6c1bfde55e1 2024-11-14T09:48:45,258 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6cc17f0b240240c29abade03ddb90ac8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/6cc17f0b240240c29abade03ddb90ac8 2024-11-14T09:48:45,259 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/f18a3d1b3fdc42febe959be7cfb5acd0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/C/f18a3d1b3fdc42febe959be7cfb5acd0 2024-11-14T09:48:45,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits/339.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061/recovered.edits/339.seqid 2024-11-14T09:48:45,262 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,262 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:48:45,262 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:48:45,263 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:48:45,267 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111408159e60f80f4febb4380c45ad338545_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111408159e60f80f4febb4380c45ad338545_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,268 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111414a0549accde4f529bdb1c0e83dd8b07_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111414a0549accde4f529bdb1c0e83dd8b07_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,269 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ff3b5c36e874a7f89b283d5e1d71c66_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141ff3b5c36e874a7f89b283d5e1d71c66_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,271 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e8b2084aab34a0297f08a4fee7716ef_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e8b2084aab34a0297f08a4fee7716ef_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,272 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114503e28e18dc24ca18e2e6c6b22bb4c09_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114503e28e18dc24ca18e2e6c6b22bb4c09_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,273 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452e4fdb9e8d14d48b62b44af8834437b_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452e4fdb9e8d14d48b62b44af8834437b_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,275 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c57388bd90b449390178031a4103ee3_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145c57388bd90b449390178031a4103ee3_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,276 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145eb5887974504b4692afeffa0c50be0b_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145eb5887974504b4692afeffa0c50be0b_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,277 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462c5330af84447fdb258e74c983909d0_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111462c5330af84447fdb258e74c983909d0_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,279 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147ecf123724e749be976f5ddc7b16b820_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147ecf123724e749be976f5ddc7b16b820_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,280 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149338518b54734eedb3888488869f1407_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149338518b54734eedb3888488869f1407_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,281 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ca499827b1546529e923aebb55e5a81_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ca499827b1546529e923aebb55e5a81_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,282 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f870657a68543ea833861e935478672_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f870657a68543ea833861e935478672_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,284 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a26a7e37970d4d37b9725fb10d1caa49_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a26a7e37970d4d37b9725fb10d1caa49_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,286 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b4e5de71d4234a0d98187ef801b63bd5_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b4e5de71d4234a0d98187ef801b63bd5_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,287 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d18d57760b084bd1ad722a63ffb51201_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d18d57760b084bd1ad722a63ffb51201_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,288 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114defd3a9a8ead4fbd851e36c3635931b2_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114defd3a9a8ead4fbd851e36c3635931b2_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,290 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb68050ec4c94889a8ce8a7de0871ed8_2b3f89fc172b9408ca73a69d8ba65061 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb68050ec4c94889a8ce8a7de0871ed8_2b3f89fc172b9408ca73a69d8ba65061 2024-11-14T09:48:45,290 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:48:45,293 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,295 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:48:45,298 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:48:45,300 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,300 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:48:45,300 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577725300"}]},"ts":"9223372036854775807"} 2024-11-14T09:48:45,303 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:48:45,303 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2b3f89fc172b9408ca73a69d8ba65061, NAME => 'TestAcidGuarantees,,1731577691057.2b3f89fc172b9408ca73a69d8ba65061.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:48:45,303 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:48:45,303 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577725303"}]},"ts":"9223372036854775807"} 2024-11-14T09:48:45,305 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:48:45,306 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-14T09:48:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T09:48:45,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:48:45,345 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:45,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 430), ProcessCount=11 (was 11), AvailableMemoryMB=5125 (was 4374) - AvailableMemoryMB LEAK? - 2024-11-14T09:48:45,365 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=416, ProcessCount=11, AvailableMemoryMB=5123 2024-11-14T09:48:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:48:45,366 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:48:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:48:45,368 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:48:45,368 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:45,368 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-14T09:48:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:45,369 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:48:45,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742069_1245 (size=959) 2024-11-14T09:48:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:45,779 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:48:45,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742070_1246 (size=53) 2024-11-14T09:48:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:46,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:48:46,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ef41cd61287331f08dbf3f70d6eeab0f, disabling compactions & flushes 2024-11-14T09:48:46,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. after waiting 0 ms 2024-11-14T09:48:46,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ef41cd61287331f08dbf3f70d6eeab0f: Waiting for close lock at 1731577726186Disabling compacts and flushes for region at 1731577726186Disabling writes for close at 1731577726186Writing region close event to WAL at 1731577726186Closed at 1731577726186 2024-11-14T09:48:46,187 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:48:46,188 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577726187"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577726187"}]},"ts":"1731577726187"} 2024-11-14T09:48:46,189 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:48:46,190 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:48:46,191 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577726190"}]},"ts":"1731577726190"} 2024-11-14T09:48:46,192 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:48:46,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, ASSIGN}] 2024-11-14T09:48:46,194 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, ASSIGN 2024-11-14T09:48:46,195 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:48:46,346 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ef41cd61287331f08dbf3f70d6eeab0f, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:46,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, ASSIGN because future has completed 2024-11-14T09:48:46,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:48:46,504 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,504 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:48:46,504 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,504 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:48:46,504 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,505 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:46,506 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,507 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef41cd61287331f08dbf3f70d6eeab0f columnFamilyName A 2024-11-14T09:48:46,507 DEBUG [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:46,508 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(327): Store=ef41cd61287331f08dbf3f70d6eeab0f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:46,508 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,510 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef41cd61287331f08dbf3f70d6eeab0f columnFamilyName B 2024-11-14T09:48:46,510 DEBUG [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:46,511 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(327): Store=ef41cd61287331f08dbf3f70d6eeab0f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:46,511 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,512 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef41cd61287331f08dbf3f70d6eeab0f columnFamilyName C 2024-11-14T09:48:46,512 DEBUG [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:48:46,513 INFO [StoreOpener-ef41cd61287331f08dbf3f70d6eeab0f-1 {}] regionserver.HStore(327): Store=ef41cd61287331f08dbf3f70d6eeab0f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:48:46,513 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,514 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,514 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,515 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,515 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,516 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:48:46,518 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,520 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:48:46,521 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened ef41cd61287331f08dbf3f70d6eeab0f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68329630, jitterRate=0.018190830945968628}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:48:46,521 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:46,522 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for ef41cd61287331f08dbf3f70d6eeab0f: Running coprocessor pre-open hook at 1731577726505Writing region info on filesystem at 1731577726505Initializing all the Stores at 1731577726505Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577726506 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577726506Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577726506Cleaning up temporary data from old regions at 1731577726515 (+9 ms)Running coprocessor post-open hooks at 1731577726521 (+6 ms)Region opened successfully at 1731577726522 (+1 ms) 2024-11-14T09:48:46,523 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., pid=61, masterSystemTime=1731577726500 2024-11-14T09:48:46,525 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,525 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:46,526 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ef41cd61287331f08dbf3f70d6eeab0f, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:46,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:48:46,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T09:48:46,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 in 180 msec 2024-11-14T09:48:46,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-14T09:48:46,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, ASSIGN in 338 msec 2024-11-14T09:48:46,534 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:48:46,535 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577726534"}]},"ts":"1731577726534"} 2024-11-14T09:48:46,537 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:48:46,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:48:46,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-14T09:48:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-14T09:48:47,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:47,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,517 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3cc2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,519 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bfd8c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,520 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:48:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:48:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d26d78d 2024-11-14T09:48:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:48:47,525 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:48:47,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,526 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,527 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52851609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,531 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360de83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d7db62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,534 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b212e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,535 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347d24da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ada2fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1f4baf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,540 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48f33beb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,541 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5485c27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,543 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16344c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,544 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5d3855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c094a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,546 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d17f271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,547 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,548 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cbf99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,549 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b40e651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,551 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d538d0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d95efc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:48:47,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:48:47,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764c0fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:48:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:48:47,555 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:48:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49756db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:48:47,555 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,555 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,555 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,555 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,556 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,556 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,556 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,556 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T09:48:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,557 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:48:47,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:47,559 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,560 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:47,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:47,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,563 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,563 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:48:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:48:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:48:47,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:48:47,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:48:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:47,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:47,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T09:48:47,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577787597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577787600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577787601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577787601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577787602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/d428da5468f64adc8cd450c5fb22fe1b is 50, key is test_row_0/A:col10/1731577727566/Put/seqid=0 2024-11-14T09:48:47,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742071_1247 (size=12001) 2024-11-14T09:48:47,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/d428da5468f64adc8cd450c5fb22fe1b 2024-11-14T09:48:47,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/275bf9b9b755410d9e1b9ce072ead46b is 50, key is test_row_0/B:col10/1731577727566/Put/seqid=0 2024-11-14T09:48:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:48:47,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742072_1248 (size=12001) 2024-11-14T09:48:47,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/275bf9b9b755410d9e1b9ce072ead46b 2024-11-14T09:48:47,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577787705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:48:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:47,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:47,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:47,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577787715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577787715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577787715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577787724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/1c34146dd50d48e5a6c4d315fbbb2309 is 50, key is test_row_0/C:col10/1731577727566/Put/seqid=0 2024-11-14T09:48:47,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742073_1249 (size=12001) 2024-11-14T09:48:47,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/1c34146dd50d48e5a6c4d315fbbb2309 2024-11-14T09:48:47,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/d428da5468f64adc8cd450c5fb22fe1b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b 2024-11-14T09:48:47,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:48:47,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/275bf9b9b755410d9e1b9ce072ead46b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b 2024-11-14T09:48:47,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:48:47,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/1c34146dd50d48e5a6c4d315fbbb2309 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309 2024-11-14T09:48:47,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T09:48:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for ef41cd61287331f08dbf3f70d6eeab0f in 236ms, sequenceid=14, compaction requested=false 2024-11-14T09:48:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:47,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T09:48:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:47,869 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:48:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:48:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1b47544e2f5b474f9b2d02a3ed6a13a7 is 50, key is test_row_0/A:col10/1731577727601/Put/seqid=0 2024-11-14T09:48:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:47,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:47,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742074_1250 (size=12001) 2024-11-14T09:48:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577787935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577787935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577787937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577787939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:47,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:47,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577787940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577788046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577788046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577788046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577788055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:48:48,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577788245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577788254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577788255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577788255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577788265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,326 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1b47544e2f5b474f9b2d02a3ed6a13a7 2024-11-14T09:48:48,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13a0357556624abca1292bb18ef0c698 is 50, key is test_row_0/B:col10/1731577727601/Put/seqid=0 2024-11-14T09:48:48,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742075_1251 (size=12001) 2024-11-14T09:48:48,385 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13a0357556624abca1292bb18ef0c698 2024-11-14T09:48:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d9102c6a42e443b7ae3fd72d80218232 is 50, key is test_row_0/C:col10/1731577727601/Put/seqid=0 2024-11-14T09:48:48,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742076_1252 (size=12001) 2024-11-14T09:48:48,450 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d9102c6a42e443b7ae3fd72d80218232 2024-11-14T09:48:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1b47544e2f5b474f9b2d02a3ed6a13a7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7 2024-11-14T09:48:48,463 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T09:48:48,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13a0357556624abca1292bb18ef0c698 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698 2024-11-14T09:48:48,471 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T09:48:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d9102c6a42e443b7ae3fd72d80218232 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232 2024-11-14T09:48:48,478 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T09:48:48,479 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for ef41cd61287331f08dbf3f70d6eeab0f in 610ms, sequenceid=38, compaction requested=false 2024-11-14T09:48:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T09:48:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T09:48:48,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T09:48:48,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 920 msec 2024-11-14T09:48:48,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 932 msec 2024-11-14T09:48:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:48,567 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:48,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:48:48,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/edb3cf013901470ea635408e2186495b is 50, key is test_row_0/A:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:48,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742077_1253 (size=12001) 2024-11-14T09:48:48,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577788597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577788599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577788602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577788602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T09:48:48,695 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:48,695 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T09:48:48,697 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:48,698 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:48,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577788705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577788715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577788715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577788715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577788755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:48,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:48,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:48,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:48,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577788914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577788924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577788925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577788925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:48,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/edb3cf013901470ea635408e2186495b 2024-11-14T09:48:49,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/f5114ec3df1f47f7a6df010188ffaa0a is 50, key is test_row_0/B:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:49,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:49,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742078_1254 (size=12001) 2024-11-14T09:48:49,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577789225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577789234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577789235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577789238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:49,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/f5114ec3df1f47f7a6df010188ffaa0a 2024-11-14T09:48:49,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/de5a159b13a74bf0990991509dce29d4 is 50, key is test_row_0/C:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:49,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742079_1255 (size=12001) 2024-11-14T09:48:49,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577789735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577789745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577789745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577789755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577789765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:49,767 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:49,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:49,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:49,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:49,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/de5a159b13a74bf0990991509dce29d4 2024-11-14T09:48:49,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/edb3cf013901470ea635408e2186495b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b 2024-11-14T09:48:49,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T09:48:49,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/f5114ec3df1f47f7a6df010188ffaa0a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a 2024-11-14T09:48:49,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T09:48:49,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/de5a159b13a74bf0990991509dce29d4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4 2024-11-14T09:48:49,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T09:48:49,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for ef41cd61287331f08dbf3f70d6eeab0f in 1300ms, sequenceid=54, compaction requested=true 2024-11-14T09:48:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:49,867 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:49,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:49,867 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:49,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:49,868 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:49,868 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:49,868 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:48:49,868 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:48:49,869 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,869 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,869 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.2 K 2024-11-14T09:48:49,869 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.2 K 2024-11-14T09:48:49,869 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting d428da5468f64adc8cd450c5fb22fe1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577727566 2024-11-14T09:48:49,869 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 275bf9b9b755410d9e1b9ce072ead46b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577727566 2024-11-14T09:48:49,869 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b47544e2f5b474f9b2d02a3ed6a13a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731577727595 2024-11-14T09:48:49,869 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a0357556624abca1292bb18ef0c698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731577727595 2024-11-14T09:48:49,869 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb3cf013901470ea635408e2186495b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:49,870 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f5114ec3df1f47f7a6df010188ffaa0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:49,877 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:49,878 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/12ae953605e3469596ed61d83e16bd5a is 50, key is test_row_0/B:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:49,880 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:49,880 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/2aa1aa3629ba49cfac417f0e6531b96b is 50, key is test_row_0/A:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:49,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742081_1257 (size=12104) 2024-11-14T09:48:49,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742080_1256 (size=12104) 2024-11-14T09:48:49,899 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/12ae953605e3469596ed61d83e16bd5a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/12ae953605e3469596ed61d83e16bd5a 2024-11-14T09:48:49,905 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into 12ae953605e3469596ed61d83e16bd5a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:49,905 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:49,905 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577729867; duration=0sec 2024-11-14T09:48:49,905 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:49,905 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:48:49,905 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:49,906 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:49,906 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:48:49,906 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,907 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.2 K 2024-11-14T09:48:49,907 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c34146dd50d48e5a6c4d315fbbb2309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731577727566 2024-11-14T09:48:49,907 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d9102c6a42e443b7ae3fd72d80218232, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731577727595 2024-11-14T09:48:49,908 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting de5a159b13a74bf0990991509dce29d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:49,914 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:49,915 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/e6dee8b4fcd7462fa6be889d0fcfa1f1 is 50, key is test_row_0/C:col10/1731577728567/Put/seqid=0 2024-11-14T09:48:49,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742082_1258 (size=12104) 2024-11-14T09:48:49,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T09:48:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:49,933 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:48:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f0123081776641aca955b5f6c41e1dd0 is 50, key is test_row_0/A:col10/1731577728600/Put/seqid=0 2024-11-14T09:48:49,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742083_1259 (size=12001) 2024-11-14T09:48:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f0123081776641aca955b5f6c41e1dd0 2024-11-14T09:48:49,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dbbf7a22c8514a6780af17acb5b8ad69 is 50, key is test_row_0/B:col10/1731577728600/Put/seqid=0 2024-11-14T09:48:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742084_1260 (size=12001) 2024-11-14T09:48:50,003 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dbbf7a22c8514a6780af17acb5b8ad69 2024-11-14T09:48:50,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/bb89c77819bb48f3b013a4c9240ba9eb is 50, key is test_row_0/C:col10/1731577728600/Put/seqid=0 2024-11-14T09:48:50,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742085_1261 (size=12001) 2024-11-14T09:48:50,018 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/bb89c77819bb48f3b013a4c9240ba9eb 2024-11-14T09:48:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f0123081776641aca955b5f6c41e1dd0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0 2024-11-14T09:48:50,030 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T09:48:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dbbf7a22c8514a6780af17acb5b8ad69 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69 2024-11-14T09:48:50,038 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T09:48:50,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/bb89c77819bb48f3b013a4c9240ba9eb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb 2024-11-14T09:48:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T09:48:50,073 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for ef41cd61287331f08dbf3f70d6eeab0f in 139ms, sequenceid=76, compaction requested=false 2024-11-14T09:48:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:50,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T09:48:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T09:48:50,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T09:48:50,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3770 sec 2024-11-14T09:48:50,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.3840 sec 2024-11-14T09:48:50,292 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/2aa1aa3629ba49cfac417f0e6531b96b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/2aa1aa3629ba49cfac417f0e6531b96b 2024-11-14T09:48:50,307 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 2aa1aa3629ba49cfac417f0e6531b96b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:50,307 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:50,307 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577729867; duration=0sec 2024-11-14T09:48:50,308 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:50,308 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:48:50,334 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/e6dee8b4fcd7462fa6be889d0fcfa1f1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6dee8b4fcd7462fa6be889d0fcfa1f1 2024-11-14T09:48:50,341 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into e6dee8b4fcd7462fa6be889d0fcfa1f1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:50,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:50,341 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577729867; duration=0sec 2024-11-14T09:48:50,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:50,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:48:50,570 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:48:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:50,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:50,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:50,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/959c6179297549228339e0ce0797c28d is 50, key is test_row_0/A:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:50,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742086_1262 (size=12001) 2024-11-14T09:48:50,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577790784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577790785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577790787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577790788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T09:48:50,855 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:50,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T09:48:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:50,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:50,857 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:50,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577790895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577790904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577790904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577790905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:50,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:51,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577791104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577791114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577791114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577791114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/959c6179297549228339e0ce0797c28d 2024-11-14T09:48:51,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ff28f735811e44bd87ee4aaaab5d3cd3 is 50, key is test_row_0/B:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:51,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742087_1263 (size=12001) 2024-11-14T09:48:51,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577791415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577791425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577791425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577791425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:51,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ff28f735811e44bd87ee4aaaab5d3cd3 2024-11-14T09:48:51,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/99c3ec4b6465476ca12a49e8991fa37e is 50, key is test_row_0/C:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742088_1264 (size=12001) 2024-11-14T09:48:51,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577791795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,797 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:48:51,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577791925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:51,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:51,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577791935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577791935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577791935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:52,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/99c3ec4b6465476ca12a49e8991fa37e 2024-11-14T09:48:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/959c6179297549228339e0ce0797c28d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d 2024-11-14T09:48:52,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-14T09:48:52,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ff28f735811e44bd87ee4aaaab5d3cd3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3 2024-11-14T09:48:52,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3, entries=150, sequenceid=89, filesize=11.7 K 2024-11-14T09:48:52,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/99c3ec4b6465476ca12a49e8991fa37e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e 2024-11-14T09:48:52,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-14T09:48:52,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ef41cd61287331f08dbf3f70d6eeab0f in 1282ms, sequenceid=89, compaction requested=true 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:52,039 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:52,039 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:52,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:52,044 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:52,044 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:48:52,044 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:52,044 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/2aa1aa3629ba49cfac417f0e6531b96b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.3 K 2024-11-14T09:48:52,045 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa1aa3629ba49cfac417f0e6531b96b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:52,045 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:52,045 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:48:52,045 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:52,045 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/12ae953605e3469596ed61d83e16bd5a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.3 K 2024-11-14T09:48:52,046 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f0123081776641aca955b5f6c41e1dd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731577728597 2024-11-14T09:48:52,046 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 959c6179297549228339e0ce0797c28d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:52,046 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ae953605e3469596ed61d83e16bd5a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:52,047 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbbf7a22c8514a6780af17acb5b8ad69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731577728597 2024-11-14T09:48:52,047 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff28f735811e44bd87ee4aaaab5d3cd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:52,055 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:52,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9ea084713dcb4247aa9aca0aa07a1fc9 is 50, key is test_row_0/A:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:52,059 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:52,060 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/14e3161361654d1c95fb695e5f304f91 is 50, key is test_row_0/B:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:52,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742089_1265 (size=12207) 2024-11-14T09:48:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742090_1266 (size=12207) 2024-11-14T09:48:52,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T09:48:52,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:52,085 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:48:52,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c914b7b219624dbaa471bdc9e660a5e2 is 50, key is test_row_0/A:col10/1731577730787/Put/seqid=0 2024-11-14T09:48:52,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742091_1267 (size=12001) 2024-11-14T09:48:52,110 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c914b7b219624dbaa471bdc9e660a5e2 2024-11-14T09:48:52,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/4ed0ad7c3f864588bee4219811fcd981 is 50, key is test_row_0/B:col10/1731577730787/Put/seqid=0 2024-11-14T09:48:52,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742092_1268 (size=12001) 2024-11-14T09:48:52,499 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9ea084713dcb4247aa9aca0aa07a1fc9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9ea084713dcb4247aa9aca0aa07a1fc9 2024-11-14T09:48:52,499 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/14e3161361654d1c95fb695e5f304f91 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14e3161361654d1c95fb695e5f304f91 2024-11-14T09:48:52,506 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into 14e3161361654d1c95fb695e5f304f91(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:52,506 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:52,506 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577732039; duration=0sec 2024-11-14T09:48:52,507 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:52,507 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:48:52,507 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:52,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:52,508 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:48:52,508 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:52,508 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6dee8b4fcd7462fa6be889d0fcfa1f1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.3 K 2024-11-14T09:48:52,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6dee8b4fcd7462fa6be889d0fcfa1f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731577727938 2024-11-14T09:48:52,509 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb89c77819bb48f3b013a4c9240ba9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731577728597 2024-11-14T09:48:52,510 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99c3ec4b6465476ca12a49e8991fa37e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:52,510 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 9ea084713dcb4247aa9aca0aa07a1fc9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:52,510 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:52,510 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577732039; duration=0sec 2024-11-14T09:48:52,510 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:52,510 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:48:52,533 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/4ed0ad7c3f864588bee4219811fcd981 2024-11-14T09:48:52,546 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:52,546 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23 is 50, key is test_row_0/C:col10/1731577730749/Put/seqid=0 2024-11-14T09:48:52,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/aaff7d7c45e141cebb0b2311b7618673 is 50, key is test_row_0/C:col10/1731577730787/Put/seqid=0 2024-11-14T09:48:52,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742094_1270 (size=12207) 2024-11-14T09:48:52,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742093_1269 (size=12001) 2024-11-14T09:48:52,579 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/aaff7d7c45e141cebb0b2311b7618673 2024-11-14T09:48:52,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c914b7b219624dbaa471bdc9e660a5e2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2 2024-11-14T09:48:52,588 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:48:52,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/4ed0ad7c3f864588bee4219811fcd981 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981 2024-11-14T09:48:52,595 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:48:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/aaff7d7c45e141cebb0b2311b7618673 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673 2024-11-14T09:48:52,602 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:48:52,603 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for ef41cd61287331f08dbf3f70d6eeab0f in 518ms, sequenceid=114, compaction requested=false 2024-11-14T09:48:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:52,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T09:48:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T09:48:52,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T09:48:52,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7470 sec 2024-11-14T09:48:52,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.7540 sec 2024-11-14T09:48:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:52,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:52,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:52,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/a7f179986d8b40e08104a53ac580e1e3 is 50, key is test_row_0/A:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:52,979 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23 2024-11-14T09:48:52,984 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into 3b84ddb5ca2341f7bc7ac19fb5ed9b23(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:52,984 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:52,984 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577732039; duration=0sec 2024-11-14T09:48:52,984 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:52,984 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:48:52,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742095_1271 (size=12001) 2024-11-14T09:48:52,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577792981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577792984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577792985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577792987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T09:48:53,005 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:53,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T09:48:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:53,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:53,010 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:53,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:53,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577793095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577793095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577793096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577793096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:53,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577793304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577793304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577793305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577793305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:53,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/a7f179986d8b40e08104a53ac580e1e3 2024-11-14T09:48:53,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/14ce33ba7020413e9ee5e2d4aaa17456 is 50, key is test_row_0/B:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:53,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742096_1272 (size=12001) 2024-11-14T09:48:53,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577793614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577793614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577793614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577793614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:53,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:53,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:53,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:53,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/14ce33ba7020413e9ee5e2d4aaa17456 2024-11-14T09:48:53,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/22b89bfad6d14006baca8b292a3d78e5 is 50, key is test_row_0/C:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742097_1273 (size=12001) 2024-11-14T09:48:53,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/22b89bfad6d14006baca8b292a3d78e5 2024-11-14T09:48:53,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/a7f179986d8b40e08104a53ac580e1e3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3 2024-11-14T09:48:53,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:48:53,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/14ce33ba7020413e9ee5e2d4aaa17456 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456 2024-11-14T09:48:53,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:48:53,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/22b89bfad6d14006baca8b292a3d78e5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5 2024-11-14T09:48:53,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T09:48:53,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for ef41cd61287331f08dbf3f70d6eeab0f in 896ms, sequenceid=127, compaction requested=true 2024-11-14T09:48:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:53,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:53,846 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:53,846 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:53,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:53,847 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:53,847 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:48:53,847 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,848 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9ea084713dcb4247aa9aca0aa07a1fc9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.4 K 2024-11-14T09:48:53,848 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:53,848 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:48:53,848 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea084713dcb4247aa9aca0aa07a1fc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:53,848 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,848 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14e3161361654d1c95fb695e5f304f91, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.4 K 2024-11-14T09:48:53,848 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c914b7b219624dbaa471bdc9e660a5e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577730763 2024-11-14T09:48:53,848 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e3161361654d1c95fb695e5f304f91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:53,849 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7f179986d8b40e08104a53ac580e1e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:53,849 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed0ad7c3f864588bee4219811fcd981, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577730763 2024-11-14T09:48:53,849 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ce33ba7020413e9ee5e2d4aaa17456, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:53,860 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:53,860 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/e4d13288141c49b988567fd1bdf3aa6d is 50, key is test_row_0/A:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:53,861 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:53,861 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e8adca6e086649a1900a3d3cba07ec0f is 50, key is test_row_0/B:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:53,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742098_1274 (size=12309) 2024-11-14T09:48:53,886 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/e4d13288141c49b988567fd1bdf3aa6d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e4d13288141c49b988567fd1bdf3aa6d 2024-11-14T09:48:53,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742099_1275 (size=12309) 2024-11-14T09:48:53,894 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into e4d13288141c49b988567fd1bdf3aa6d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:53,894 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:53,894 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577733846; duration=0sec 2024-11-14T09:48:53,894 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:53,894 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:48:53,894 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:53,896 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:53,896 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:48:53,896 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,896 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.4 K 2024-11-14T09:48:53,897 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b84ddb5ca2341f7bc7ac19fb5ed9b23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731577730749 2024-11-14T09:48:53,898 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaff7d7c45e141cebb0b2311b7618673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577730763 2024-11-14T09:48:53,899 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b89bfad6d14006baca8b292a3d78e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:53,899 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e8adca6e086649a1900a3d3cba07ec0f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e8adca6e086649a1900a3d3cba07ec0f 2024-11-14T09:48:53,904 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into e8adca6e086649a1900a3d3cba07ec0f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:53,904 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:53,904 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577733846; duration=0sec 2024-11-14T09:48:53,904 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:53,904 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:48:53,907 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:53,908 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/373de8dff98a45b3951fe3eef899cc38 is 50, key is test_row_0/C:col10/1731577732948/Put/seqid=0 2024-11-14T09:48:53,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742100_1276 (size=12309) 2024-11-14T09:48:53,921 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/373de8dff98a45b3951fe3eef899cc38 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/373de8dff98a45b3951fe3eef899cc38 2024-11-14T09:48:53,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T09:48:53,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:53,928 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:48:53,928 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into 373de8dff98a45b3951fe3eef899cc38(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:53,928 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:53,928 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577733847; duration=0sec 2024-11-14T09:48:53,929 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:53,929 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:48:53,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/28d1344bab314284aa1527917b9d94bc is 50, key is test_row_0/A:col10/1731577732979/Put/seqid=0 2024-11-14T09:48:53,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742101_1277 (size=12151) 2024-11-14T09:48:53,943 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/28d1344bab314284aa1527917b9d94bc 2024-11-14T09:48:53,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ed206e881fb14020aedb4ea1723f1338 is 50, key is test_row_0/B:col10/1731577732979/Put/seqid=0 2024-11-14T09:48:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742102_1278 (size=12151) 2024-11-14T09:48:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:54,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577794175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577794175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577794176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577794180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577794285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577794285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577794285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577794294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,355 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ed206e881fb14020aedb4ea1723f1338 2024-11-14T09:48:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/38495a16beb742dba5959cdff6574288 is 50, key is test_row_0/C:col10/1731577732979/Put/seqid=0 2024-11-14T09:48:54,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742103_1279 (size=12151) 2024-11-14T09:48:54,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577794495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577794495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577794495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577794505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,769 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/38495a16beb742dba5959cdff6574288 2024-11-14T09:48:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/28d1344bab314284aa1527917b9d94bc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc 2024-11-14T09:48:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:48:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ed206e881fb14020aedb4ea1723f1338 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338 2024-11-14T09:48:54,782 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:48:54,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/38495a16beb742dba5959cdff6574288 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288 2024-11-14T09:48:54,787 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:48:54,788 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for ef41cd61287331f08dbf3f70d6eeab0f in 860ms, sequenceid=156, compaction requested=false 2024-11-14T09:48:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T09:48:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T09:48:54,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T09:48:54,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-14T09:48:54,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.7940 sec 2024-11-14T09:48:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:54,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:54,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:48:54,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/bea1199794af46e9be45ef2fe219dc9e is 50, key is test_row_0/A:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:54,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577794831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742104_1280 (size=14541) 2024-11-14T09:48:54,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577794835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577794836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577794835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577794945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577794945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577794945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577794945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T09:48:55,155 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:55,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T09:48:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:55,157 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577795155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:55,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:55,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577795155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577795155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577795155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/bea1199794af46e9be45ef2fe219dc9e 2024-11-14T09:48:55,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/d07024bb54de4c4091a106796d88e893 is 50, key is test_row_0/B:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:55,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742105_1281 (size=12151) 2024-11-14T09:48:55,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/d07024bb54de4c4091a106796d88e893 2024-11-14T09:48:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:55,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6a15c1b624b64d78a07632898c525ab3 is 50, key is test_row_0/C:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:55,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742106_1282 (size=12151) 2024-11-14T09:48:55,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:48:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:48:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577795466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577795466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577795466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577795466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:48:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:55,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6a15c1b624b64d78a07632898c525ab3 2024-11-14T09:48:55,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/bea1199794af46e9be45ef2fe219dc9e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e 2024-11-14T09:48:55,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e, entries=200, sequenceid=170, filesize=14.2 K 2024-11-14T09:48:55,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/d07024bb54de4c4091a106796d88e893 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893 2024-11-14T09:48:55,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T09:48:55,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6a15c1b624b64d78a07632898c525ab3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3 2024-11-14T09:48:55,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T09:48:55,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ef41cd61287331f08dbf3f70d6eeab0f in 899ms, sequenceid=170, compaction requested=true 2024-11-14T09:48:55,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:55,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:55,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:55,706 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:55,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:55,707 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:55,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:55,710 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:55,710 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:55,710 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:48:55,710 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:48:55,710 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,710 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,710 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e4d13288141c49b988567fd1bdf3aa6d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=38.1 K 2024-11-14T09:48:55,710 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e8adca6e086649a1900a3d3cba07ec0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.8 K 2024-11-14T09:48:55,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e8adca6e086649a1900a3d3cba07ec0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:55,711 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d13288141c49b988567fd1bdf3aa6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:55,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ed206e881fb14020aedb4ea1723f1338, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577732979 2024-11-14T09:48:55,711 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28d1344bab314284aa1527917b9d94bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577732979 2024-11-14T09:48:55,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d07024bb54de4c4091a106796d88e893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:55,712 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea1199794af46e9be45ef2fe219dc9e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:55,720 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:55,721 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1be8acd258e3482d8e42f2391932b773 is 50, key is test_row_0/A:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:55,722 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:55,723 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/faafdb6f65eb418299747ec28d08ae21 is 50, key is test_row_0/B:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:55,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742108_1284 (size=12561) 2024-11-14T09:48:55,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742107_1283 (size=12561) 2024-11-14T09:48:55,740 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/faafdb6f65eb418299747ec28d08ae21 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/faafdb6f65eb418299747ec28d08ae21 2024-11-14T09:48:55,743 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1be8acd258e3482d8e42f2391932b773 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1be8acd258e3482d8e42f2391932b773 2024-11-14T09:48:55,758 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into faafdb6f65eb418299747ec28d08ae21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:55,758 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:55,758 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577735706; duration=0sec 2024-11-14T09:48:55,758 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:55,758 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:48:55,758 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:55,759 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 1be8acd258e3482d8e42f2391932b773(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:55,759 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:55,759 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577735706; duration=0sec 2024-11-14T09:48:55,759 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:55,759 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:55,759 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:48:55,760 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:48:55,760 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,760 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/373de8dff98a45b3951fe3eef899cc38, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=35.8 K 2024-11-14T09:48:55,760 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 373de8dff98a45b3951fe3eef899cc38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731577732945 2024-11-14T09:48:55,760 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 38495a16beb742dba5959cdff6574288, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577732979 2024-11-14T09:48:55,761 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a15c1b624b64d78a07632898c525ab3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:55,769 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:55,770 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/b9241a408e1b4a0587f8c9ea80ddac2a is 50, key is test_row_0/C:col10/1731577734132/Put/seqid=0 2024-11-14T09:48:55,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T09:48:55,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:55,773 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:48:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f56e983c5f474fefbd1b1154390fd121 is 50, key is test_row_0/A:col10/1731577734831/Put/seqid=0 2024-11-14T09:48:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:55,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742109_1285 (size=12561) 2024-11-14T09:48:55,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742110_1286 (size=12151) 2024-11-14T09:48:55,798 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f56e983c5f474fefbd1b1154390fd121 2024-11-14T09:48:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dc6f29de9d524292bb168e0c89a5b471 is 50, key is test_row_0/B:col10/1731577734831/Put/seqid=0 2024-11-14T09:48:55,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:55,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742111_1287 (size=12151) 2024-11-14T09:48:55,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577795845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577795954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577795974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577795985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577795985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577795985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:55,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:56,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:56,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577796164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:56,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:56,200 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/b9241a408e1b4a0587f8c9ea80ddac2a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b9241a408e1b4a0587f8c9ea80ddac2a 2024-11-14T09:48:56,204 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into b9241a408e1b4a0587f8c9ea80ddac2a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:56,204 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:56,205 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577735707; duration=0sec 2024-11-14T09:48:56,205 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:56,205 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:48:56,215 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dc6f29de9d524292bb168e0c89a5b471 2024-11-14T09:48:56,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/b2fe567167d74566bc59e93192cd0f85 is 50, key is test_row_0/C:col10/1731577734831/Put/seqid=0 2024-11-14T09:48:56,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742112_1288 (size=12151) 2024-11-14T09:48:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577796474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:56,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:56,646 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/b2fe567167d74566bc59e93192cd0f85 2024-11-14T09:48:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/f56e983c5f474fefbd1b1154390fd121 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121 2024-11-14T09:48:56,661 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T09:48:56,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/dc6f29de9d524292bb168e0c89a5b471 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471 2024-11-14T09:48:56,666 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T09:48:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/b2fe567167d74566bc59e93192cd0f85 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85 2024-11-14T09:48:56,672 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85, entries=150, sequenceid=194, filesize=11.9 K 2024-11-14T09:48:56,672 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ef41cd61287331f08dbf3f70d6eeab0f in 899ms, sequenceid=194, compaction requested=false 2024-11-14T09:48:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T09:48:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T09:48:56,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T09:48:56,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5160 sec 2024-11-14T09:48:56,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5210 sec 2024-11-14T09:48:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:56,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:56,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:48:56,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9b0d6d20f82d444e8b5a63d9715ff216 is 50, key is test_row_0/A:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:57,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742113_1289 (size=14541) 2024-11-14T09:48:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577797023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577797025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577797027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577797028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577797029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577797134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577797135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577797135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577797135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577797135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T09:48:57,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:57,305 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T09:48:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:57,306 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:57,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:57,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577797345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9b0d6d20f82d444e8b5a63d9715ff216 2024-11-14T09:48:57,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce9681bf747f4d27b46e66469d380024 is 50, key is test_row_0/B:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:57,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742114_1290 (size=12151) 2024-11-14T09:48:57,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:57,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577797655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577797656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577797656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577797656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577797656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:57,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce9681bf747f4d27b46e66469d380024 2024-11-14T09:48:57,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d405d83e63f64cb8be1aa61ad7a89bac is 50, key is test_row_0/C:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:57,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742115_1291 (size=12151) 2024-11-14T09:48:57,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:57,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:58,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:58,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577798165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:58,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:58,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577798174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:58,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577798174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:58,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577798175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:58,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577798175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:58,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:58,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d405d83e63f64cb8be1aa61ad7a89bac 2024-11-14T09:48:58,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9b0d6d20f82d444e8b5a63d9715ff216 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216 2024-11-14T09:48:58,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216, entries=200, sequenceid=210, filesize=14.2 K 2024-11-14T09:48:58,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce9681bf747f4d27b46e66469d380024 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024 2024-11-14T09:48:58,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T09:48:58,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d405d83e63f64cb8be1aa61ad7a89bac as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac 2024-11-14T09:48:58,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T09:48:58,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for ef41cd61287331f08dbf3f70d6eeab0f in 1287ms, sequenceid=210, compaction requested=true 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:58,276 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:58,276 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:48:58,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:58,277 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:58,277 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:48:58,277 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,277 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1be8acd258e3482d8e42f2391932b773, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=38.3 K 2024-11-14T09:48:58,278 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1be8acd258e3482d8e42f2391932b773, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:58,278 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:58,278 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:48:58,278 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,278 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/faafdb6f65eb418299747ec28d08ae21, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.0 K 2024-11-14T09:48:58,278 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56e983c5f474fefbd1b1154390fd121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731577734831 2024-11-14T09:48:58,279 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting faafdb6f65eb418299747ec28d08ae21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:58,279 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0d6d20f82d444e8b5a63d9715ff216, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735820 2024-11-14T09:48:58,279 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting dc6f29de9d524292bb168e0c89a5b471, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731577734831 2024-11-14T09:48:58,281 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ce9681bf747f4d27b46e66469d380024, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735831 2024-11-14T09:48:58,291 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:58,291 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/fc19db8cfeb4430e8ca7dfccfef2adc8 is 50, key is test_row_0/B:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:58,294 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:58,295 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/94a0f157d8fb49c3b41809487c171c88 is 50, key is test_row_0/A:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:58,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742116_1292 (size=12663) 2024-11-14T09:48:58,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742117_1293 (size=12663) 2024-11-14T09:48:58,336 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/94a0f157d8fb49c3b41809487c171c88 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/94a0f157d8fb49c3b41809487c171c88 2024-11-14T09:48:58,341 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 94a0f157d8fb49c3b41809487c171c88(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:58,341 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:58,341 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577738276; duration=0sec 2024-11-14T09:48:58,341 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:48:58,342 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:48:58,342 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:48:58,342 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:48:58,343 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:48:58,343 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,343 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b9241a408e1b4a0587f8c9ea80ddac2a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.0 K 2024-11-14T09:48:58,343 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9241a408e1b4a0587f8c9ea80ddac2a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731577734132 2024-11-14T09:48:58,343 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2fe567167d74566bc59e93192cd0f85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731577734831 2024-11-14T09:48:58,344 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting d405d83e63f64cb8be1aa61ad7a89bac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735831 2024-11-14T09:48:58,349 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:48:58,350 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6fe436bb98fc4853a2916e0120621693 is 50, key is test_row_0/C:col10/1731577736986/Put/seqid=0 2024-11-14T09:48:58,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742118_1294 (size=12663) 2024-11-14T09:48:58,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T09:48:58,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,379 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:48:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c5532690a69e49fc90e5fd23a3206aa4 is 50, key is test_row_0/A:col10/1731577737027/Put/seqid=0 2024-11-14T09:48:58,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742119_1295 (size=12151) 2024-11-14T09:48:58,407 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c5532690a69e49fc90e5fd23a3206aa4 2024-11-14T09:48:58,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/46e93ece96c54cc088bbcbaebd5d2b60 is 50, key is test_row_0/B:col10/1731577737027/Put/seqid=0 2024-11-14T09:48:58,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742120_1296 (size=12151) 2024-11-14T09:48:58,426 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/46e93ece96c54cc088bbcbaebd5d2b60 2024-11-14T09:48:58,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/2ecaab52a19b4a78bf6f8df7583c6579 is 50, key is test_row_0/C:col10/1731577737027/Put/seqid=0 2024-11-14T09:48:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:58,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742121_1297 (size=12151) 2024-11-14T09:48:58,707 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/fc19db8cfeb4430e8ca7dfccfef2adc8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/fc19db8cfeb4430e8ca7dfccfef2adc8 2024-11-14T09:48:58,711 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into fc19db8cfeb4430e8ca7dfccfef2adc8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:58,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:58,711 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577738276; duration=0sec 2024-11-14T09:48:58,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:58,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:48:58,758 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6fe436bb98fc4853a2916e0120621693 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fe436bb98fc4853a2916e0120621693 2024-11-14T09:48:58,763 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into 6fe436bb98fc4853a2916e0120621693(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:48:58,763 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:58,763 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577738276; duration=0sec 2024-11-14T09:48:58,763 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:48:58,763 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:48:58,851 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/2ecaab52a19b4a78bf6f8df7583c6579 2024-11-14T09:48:58,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/c5532690a69e49fc90e5fd23a3206aa4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4 2024-11-14T09:48:58,869 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:48:58,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/46e93ece96c54cc088bbcbaebd5d2b60 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60 2024-11-14T09:48:58,877 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:48:58,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/2ecaab52a19b4a78bf6f8df7583c6579 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579 2024-11-14T09:48:58,892 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579, entries=150, sequenceid=233, filesize=11.9 K 2024-11-14T09:48:58,893 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for ef41cd61287331f08dbf3f70d6eeab0f in 514ms, sequenceid=233, compaction requested=false 2024-11-14T09:48:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:48:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:58,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T09:48:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T09:48:58,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T09:48:58,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5880 sec 2024-11-14T09:48:58,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5940 sec 2024-11-14T09:48:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:48:59,203 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:48:59,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:48:59,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/b679425e15ea4720ae2a7f01201ca32c is 50, key is test_row_0/A:col10/1731577739201/Put/seqid=0 2024-11-14T09:48:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577799261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577799262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577799264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577799265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577799265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742122_1298 (size=12151) 2024-11-14T09:48:59,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/b679425e15ea4720ae2a7f01201ca32c 2024-11-14T09:48:59,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/66f95a801a3342ddab645c5fe93a052e is 50, key is test_row_0/B:col10/1731577739201/Put/seqid=0 2024-11-14T09:48:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742123_1299 (size=12151) 2024-11-14T09:48:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577799375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577799375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577799375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577799375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577799375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T09:48:59,465 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:48:59,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T09:48:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:48:59,467 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:48:59,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:48:59,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:48:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:48:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577799586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577799585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577799586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577799586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577799586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:48:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/66f95a801a3342ddab645c5fe93a052e 2024-11-14T09:48:59,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6fd9b5fc549d4934b94605d84dd768c9 is 50, key is test_row_0/C:col10/1731577739201/Put/seqid=0 2024-11-14T09:48:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742124_1300 (size=12151) 2024-11-14T09:48:59,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:48:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:48:59,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577799895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577799895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577799895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577799895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:48:59,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577799895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:48:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:48:59,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:48:59,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:48:59,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:48:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:48:59,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:48:59,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:00,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:49:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:49:00,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6fd9b5fc549d4934b94605d84dd768c9 2024-11-14T09:49:00,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/b679425e15ea4720ae2a7f01201ca32c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c 2024-11-14T09:49:00,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T09:49:00,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/66f95a801a3342ddab645c5fe93a052e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e 2024-11-14T09:49:00,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T09:49:00,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6fd9b5fc549d4934b94605d84dd768c9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9 2024-11-14T09:49:00,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T09:49:00,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for ef41cd61287331f08dbf3f70d6eeab0f in 970ms, sequenceid=247, compaction requested=true 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:00,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:49:00,174 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:49:00,175 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,175 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,175 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/fc19db8cfeb4430e8ca7dfccfef2adc8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.1 K 2024-11-14T09:49:00,175 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/94a0f157d8fb49c3b41809487c171c88, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.1 K 2024-11-14T09:49:00,175 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting fc19db8cfeb4430e8ca7dfccfef2adc8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735831 2024-11-14T09:49:00,175 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a0f157d8fb49c3b41809487c171c88, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735831 2024-11-14T09:49:00,175 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5532690a69e49fc90e5fd23a3206aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577737025 2024-11-14T09:49:00,175 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e93ece96c54cc088bbcbaebd5d2b60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577737025 2024-11-14T09:49:00,176 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b679425e15ea4720ae2a7f01201ca32c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:00,176 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f95a801a3342ddab645c5fe93a052e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:00,182 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:00,182 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:00,182 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/2418f15741de4e69b80df71a0c3cbc51 is 50, key is test_row_0/B:col10/1731577739201/Put/seqid=0 2024-11-14T09:49:00,182 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/0e91d3304f0049209da359d4148d67c6 is 50, key is test_row_0/A:col10/1731577739201/Put/seqid=0 2024-11-14T09:49:00,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742125_1301 (size=12765) 2024-11-14T09:49:00,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742126_1302 (size=12765) 2024-11-14T09:49:00,197 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/0e91d3304f0049209da359d4148d67c6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/0e91d3304f0049209da359d4148d67c6 2024-11-14T09:49:00,201 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 0e91d3304f0049209da359d4148d67c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:00,201 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:00,201 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577740174; duration=0sec 2024-11-14T09:49:00,201 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:00,201 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:49:00,201 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:00,202 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:00,202 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:49:00,202 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,202 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fe436bb98fc4853a2916e0120621693, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.1 K 2024-11-14T09:49:00,204 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fe436bb98fc4853a2916e0120621693, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731577735831 2024-11-14T09:49:00,204 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ecaab52a19b4a78bf6f8df7583c6579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731577737025 2024-11-14T09:49:00,205 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fd9b5fc549d4934b94605d84dd768c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:00,211 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:00,212 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/0aeb42a7a9234de2ac6ee0d7269ddaf8 is 50, key is test_row_0/C:col10/1731577739201/Put/seqid=0 2024-11-14T09:49:00,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742127_1303 (size=12765) 2024-11-14T09:49:00,219 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/0aeb42a7a9234de2ac6ee0d7269ddaf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/0aeb42a7a9234de2ac6ee0d7269ddaf8 2024-11-14T09:49:00,223 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into 0aeb42a7a9234de2ac6ee0d7269ddaf8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:00,223 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:00,223 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577740174; duration=0sec 2024-11-14T09:49:00,223 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:00,223 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:49:00,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T09:49:00,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:00,231 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:49:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/5ce04684cd184075a190dfa1282d4be7 is 50, key is test_row_0/A:col10/1731577739263/Put/seqid=0 2024-11-14T09:49:00,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742128_1304 (size=12301) 2024-11-14T09:49:00,239 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/5ce04684cd184075a190dfa1282d4be7 2024-11-14T09:49:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/196afc0ce749423782f6a6cf2667f5e1 is 50, key is test_row_0/B:col10/1731577739263/Put/seqid=0 2024-11-14T09:49:00,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742129_1305 (size=12301) 2024-11-14T09:49:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:00,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577800413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577800412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577800415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577800418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577800419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577800525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577800525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577800525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577800534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,596 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/2418f15741de4e69b80df71a0c3cbc51 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/2418f15741de4e69b80df71a0c3cbc51 2024-11-14T09:49:00,600 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into 2418f15741de4e69b80df71a0c3cbc51(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:00,600 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:00,600 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577740174; duration=0sec 2024-11-14T09:49:00,601 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:00,601 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:49:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:49:00,651 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/196afc0ce749423782f6a6cf2667f5e1 2024-11-14T09:49:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6688fabf1ddf4967bd4d39c51459a9e8 is 50, key is test_row_0/C:col10/1731577739263/Put/seqid=0 2024-11-14T09:49:00,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742130_1306 (size=12301) 2024-11-14T09:49:00,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577800735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577800735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577800735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577800744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577801045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577801045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577801045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577801055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,070 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6688fabf1ddf4967bd4d39c51459a9e8 2024-11-14T09:49:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/5ce04684cd184075a190dfa1282d4be7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7 2024-11-14T09:49:01,078 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T09:49:01,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/196afc0ce749423782f6a6cf2667f5e1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1 2024-11-14T09:49:01,082 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T09:49:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/6688fabf1ddf4967bd4d39c51459a9e8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8 2024-11-14T09:49:01,086 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T09:49:01,087 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for ef41cd61287331f08dbf3f70d6eeab0f in 856ms, sequenceid=274, compaction requested=false 2024-11-14T09:49:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T09:49:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T09:49:01,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T09:49:01,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-14T09:49:01,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6250 sec 2024-11-14T09:49:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:01,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:01,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:49:01,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/03b6669ee3494100b885898879f30395 is 50, key is test_row_0/A:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:01,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742131_1307 (size=12301) 2024-11-14T09:49:01,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577801507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577801555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577801555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577801555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577801564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T09:49:01,615 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:01,615 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T09:49:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577801615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,617 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,618 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:01,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:01,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:01,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:01,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577801824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:01,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/03b6669ee3494100b885898879f30395 2024-11-14T09:49:01,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce2b57591a9a423680dd017fa4248e78 is 50, key is test_row_0/B:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:01,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742132_1308 (size=12301) 2024-11-14T09:49:01,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:01,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:01,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:02,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:02,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577802135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:02,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce2b57591a9a423680dd017fa4248e78 2024-11-14T09:49:02,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/196863a7780c415595fcc6532430d3e6 is 50, key is test_row_0/C:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:02,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742133_1309 (size=12301) 2024-11-14T09:49:02,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577802564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577802564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,566 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:02,566 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577802574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577802574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,576 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,577 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577802645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:02,646 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:02,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/196863a7780c415595fcc6532430d3e6 2024-11-14T09:49:02,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/03b6669ee3494100b885898879f30395 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395 2024-11-14T09:49:02,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T09:49:02,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/ce2b57591a9a423680dd017fa4248e78 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78 2024-11-14T09:49:02,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T09:49:02,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/196863a7780c415595fcc6532430d3e6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6 2024-11-14T09:49:02,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T09:49:02,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for ef41cd61287331f08dbf3f70d6eeab0f in 1249ms, sequenceid=288, compaction requested=true 2024-11-14T09:49:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:02,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,687 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:02,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:02,687 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:02,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:02,688 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:02,688 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:49:02,688 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,688 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/0e91d3304f0049209da359d4148d67c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.5 K 2024-11-14T09:49:02,689 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e91d3304f0049209da359d4148d67c6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:02,689 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:02,689 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:49:02,689 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,689 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/2418f15741de4e69b80df71a0c3cbc51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.5 K 2024-11-14T09:49:02,690 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce04684cd184075a190dfa1282d4be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731577739260 2024-11-14T09:49:02,690 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 2418f15741de4e69b80df71a0c3cbc51, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:02,690 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 196afc0ce749423782f6a6cf2667f5e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731577739260 2024-11-14T09:49:02,690 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03b6669ee3494100b885898879f30395, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:02,691 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2b57591a9a423680dd017fa4248e78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:02,697 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:02,698 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/4ff56d4cd154426b8a8b9401a8f4af80 is 50, key is test_row_0/B:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:02,702 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:02,703 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/fae734cd67e94611a1723be49b3a8da8 is 50, key is test_row_0/A:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:02,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742134_1310 (size=13017) 2024-11-14T09:49:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742135_1311 (size=13017) 2024-11-14T09:49:02,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:02,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T09:49:02,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:02,840 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T09:49:02,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/409ceb0b015f4fbc8743c292d21ba595 is 50, key is test_row_0/A:col10/1731577741502/Put/seqid=0 2024-11-14T09:49:02,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742136_1312 (size=12301) 2024-11-14T09:49:03,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/4ff56d4cd154426b8a8b9401a8f4af80 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ff56d4cd154426b8a8b9401a8f4af80 2024-11-14T09:49:03,117 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into 4ff56d4cd154426b8a8b9401a8f4af80(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:03,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:03,117 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577742687; duration=0sec 2024-11-14T09:49:03,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:03,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:49:03,117 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:03,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:03,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:49:03,118 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:03,118 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/0aeb42a7a9234de2ac6ee0d7269ddaf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.5 K 2024-11-14T09:49:03,118 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aeb42a7a9234de2ac6ee0d7269ddaf8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731577739182 2024-11-14T09:49:03,119 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6688fabf1ddf4967bd4d39c51459a9e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731577739260 2024-11-14T09:49:03,119 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 196863a7780c415595fcc6532430d3e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:03,121 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/fae734cd67e94611a1723be49b3a8da8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/fae734cd67e94611a1723be49b3a8da8 2024-11-14T09:49:03,126 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:03,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/e6f81f09613c42f0b4a051b51b019505 is 50, key is test_row_0/C:col10/1731577741436/Put/seqid=0 2024-11-14T09:49:03,128 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into fae734cd67e94611a1723be49b3a8da8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:03,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:03,128 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577742687; duration=0sec 2024-11-14T09:49:03,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:03,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:49:03,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742137_1313 (size=13017) 2024-11-14T09:49:03,308 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/409ceb0b015f4fbc8743c292d21ba595 2024-11-14T09:49:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/b83c420c94e545a382ee1f46ce57fc09 is 50, key is test_row_0/B:col10/1731577741502/Put/seqid=0 2024-11-14T09:49:03,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742138_1314 (size=12301) 2024-11-14T09:49:03,539 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/e6f81f09613c42f0b4a051b51b019505 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6f81f09613c42f0b4a051b51b019505 2024-11-14T09:49:03,545 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into e6f81f09613c42f0b4a051b51b019505(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:03,545 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:03,545 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577742687; duration=0sec 2024-11-14T09:49:03,545 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:03,545 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:49:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:03,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577803693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:03,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:03,720 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/b83c420c94e545a382ee1f46ce57fc09 2024-11-14T09:49:03,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/257127e5b54741768864276db156730a is 50, key is test_row_0/C:col10/1731577741502/Put/seqid=0 2024-11-14T09:49:03,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742139_1315 (size=12301) 2024-11-14T09:49:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:03,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577803805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577804014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,130 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/257127e5b54741768864276db156730a 2024-11-14T09:49:04,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/409ceb0b015f4fbc8743c292d21ba595 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595 2024-11-14T09:49:04,142 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T09:49:04,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/b83c420c94e545a382ee1f46ce57fc09 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09 2024-11-14T09:49:04,149 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T09:49:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/257127e5b54741768864276db156730a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a 2024-11-14T09:49:04,162 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T09:49:04,163 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for ef41cd61287331f08dbf3f70d6eeab0f in 1323ms, sequenceid=311, compaction requested=false 2024-11-14T09:49:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:04,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T09:49:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T09:49:04,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T09:49:04,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5470 sec 2024-11-14T09:49:04,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.5540 sec 2024-11-14T09:49:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:04,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:04,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:49:04,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/8a6d19fc4c49459881c4683ae069aec6 is 50, key is test_row_0/A:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:04,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742140_1316 (size=14741) 2024-11-14T09:49:04,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577804396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577804505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45642 deadline: 1731577804574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,577 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45650 deadline: 1731577804585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,587 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45582 deadline: 1731577804595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,597 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45598 deadline: 1731577804604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,606 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in ef41cd61287331f08dbf3f70d6eeab0f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577804715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:04,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:04,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/8a6d19fc4c49459881c4683ae069aec6 2024-11-14T09:49:04,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/8fc8096b146f45f3bc6437fc061b8abb is 50, key is test_row_0/B:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742141_1317 (size=12301) 2024-11-14T09:49:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:05,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577805025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:05,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/8fc8096b146f45f3bc6437fc061b8abb 2024-11-14T09:49:05,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8e82354642724763bc2592b2facbca51 is 50, key is test_row_0/C:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:05,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742142_1318 (size=12301) 2024-11-14T09:49:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577805534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:05,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8e82354642724763bc2592b2facbca51 2024-11-14T09:49:05,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/8a6d19fc4c49459881c4683ae069aec6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6 2024-11-14T09:49:05,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6, entries=200, sequenceid=328, filesize=14.4 K 2024-11-14T09:49:05,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/8fc8096b146f45f3bc6437fc061b8abb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb 2024-11-14T09:49:05,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T09:49:05,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8e82354642724763bc2592b2facbca51 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51 2024-11-14T09:49:05,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T09:49:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for ef41cd61287331f08dbf3f70d6eeab0f in 1253ms, sequenceid=328, compaction requested=true 2024-11-14T09:49:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:05,581 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:05,581 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:05,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:05,582 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:05,582 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:05,582 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:49:05,582 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:49:05,582 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:05,582 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:05,582 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ff56d4cd154426b8a8b9401a8f4af80, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.7 K 2024-11-14T09:49:05,582 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/fae734cd67e94611a1723be49b3a8da8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=39.1 K 2024-11-14T09:49:05,583 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae734cd67e94611a1723be49b3a8da8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:05,583 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ff56d4cd154426b8a8b9401a8f4af80, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:05,583 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 409ceb0b015f4fbc8743c292d21ba595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731577741493 2024-11-14T09:49:05,583 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b83c420c94e545a382ee1f46ce57fc09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731577741493 2024-11-14T09:49:05,583 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6d19fc4c49459881c4683ae069aec6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743675 2024-11-14T09:49:05,584 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc8096b146f45f3bc6437fc061b8abb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743684 2024-11-14T09:49:05,610 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:05,614 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e1e4c4e203a04494a2e697dd74e71118 is 50, key is test_row_0/B:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:05,623 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:05,623 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/32bf6f946d66445c8178ae4591f93e6f is 50, key is test_row_0/A:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742143_1319 (size=13119) 2024-11-14T09:49:05,656 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e1e4c4e203a04494a2e697dd74e71118 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e1e4c4e203a04494a2e697dd74e71118 2024-11-14T09:49:05,663 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into e1e4c4e203a04494a2e697dd74e71118(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:05,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:05,663 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577745581; duration=0sec 2024-11-14T09:49:05,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:05,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:49:05,663 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:05,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742144_1320 (size=13119) 2024-11-14T09:49:05,666 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:05,666 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:49:05,666 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:05,666 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6f81f09613c42f0b4a051b51b019505, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.7 K 2024-11-14T09:49:05,667 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f81f09613c42f0b4a051b51b019505, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731577740410 2024-11-14T09:49:05,667 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 257127e5b54741768864276db156730a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731577741493 2024-11-14T09:49:05,667 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e82354642724763bc2592b2facbca51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743684 2024-11-14T09:49:05,679 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:05,679 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/1774fa222931441f96bb8a9f373d7935 is 50, key is test_row_0/C:col10/1731577743691/Put/seqid=0 2024-11-14T09:49:05,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742145_1321 (size=13119) 2024-11-14T09:49:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T09:49:05,785 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:05,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T09:49:05,787 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:05,787 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:05,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:05,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T09:49:05,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:05,940 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:05,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/15b45ad84e604013a822f92f227d4f60 is 50, key is test_row_0/A:col10/1731577744395/Put/seqid=0 2024-11-14T09:49:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742146_1322 (size=12301) 2024-11-14T09:49:06,070 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/32bf6f946d66445c8178ae4591f93e6f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32bf6f946d66445c8178ae4591f93e6f 2024-11-14T09:49:06,074 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 32bf6f946d66445c8178ae4591f93e6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:06,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:06,074 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577745580; duration=0sec 2024-11-14T09:49:06,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:06,074 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:49:06,089 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/1774fa222931441f96bb8a9f373d7935 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1774fa222931441f96bb8a9f373d7935 2024-11-14T09:49:06,094 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into 1774fa222931441f96bb8a9f373d7935(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:06,094 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:06,094 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577745581; duration=0sec 2024-11-14T09:49:06,094 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:06,094 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:49:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:06,350 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/15b45ad84e604013a822f92f227d4f60 2024-11-14T09:49:06,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/7a60c898196547df8e9cc51adcfe1aa0 is 50, key is test_row_0/B:col10/1731577744395/Put/seqid=0 2024-11-14T09:49:06,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742147_1323 (size=12301) 2024-11-14T09:49:06,376 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:49:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:06,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:06,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. as already flushing 2024-11-14T09:49:06,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577806577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:06,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:06,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577806685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:06,760 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/7a60c898196547df8e9cc51adcfe1aa0 2024-11-14T09:49:06,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/dec0b9fc8d904b33b6c7d82f7d4327da is 50, key is test_row_0/C:col10/1731577744395/Put/seqid=0 2024-11-14T09:49:06,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742148_1324 (size=12301) 2024-11-14T09:49:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577806894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:06,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:07,169 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/dec0b9fc8d904b33b6c7d82f7d4327da 2024-11-14T09:49:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/15b45ad84e604013a822f92f227d4f60 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60 2024-11-14T09:49:07,176 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60, entries=150, sequenceid=351, filesize=12.0 K 2024-11-14T09:49:07,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/7a60c898196547df8e9cc51adcfe1aa0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0 2024-11-14T09:49:07,180 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0, entries=150, sequenceid=351, filesize=12.0 K 2024-11-14T09:49:07,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/dec0b9fc8d904b33b6c7d82f7d4327da as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da 2024-11-14T09:49:07,184 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da, entries=150, sequenceid=351, filesize=12.0 K 2024-11-14T09:49:07,185 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for ef41cd61287331f08dbf3f70d6eeab0f in 1245ms, sequenceid=351, compaction requested=false 2024-11-14T09:49:07,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:07,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:07,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T09:49:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T09:49:07,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T09:49:07,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-14T09:49:07,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4040 sec 2024-11-14T09:49:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:07,206 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:07,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:07,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/32ec8defdbc247c99776d9e85f10af05 is 50, key is test_row_0/A:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:07,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742149_1325 (size=14741) 2024-11-14T09:49:07,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577807275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:07,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577807385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:07,563 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-14T09:49:07,564 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:07,564 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,564 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:07,565 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-14T09:49:07,565 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:07,565 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,565 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:07,566 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-14T09:49:07,566 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:07,566 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,566 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:07,566 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-14T09:49:07,566 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:07,566 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,566 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:07,568 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-14T09:49:07,568 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:07,568 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,568 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:07,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:07,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577807595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:07,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:07,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/32ec8defdbc247c99776d9e85f10af05 2024-11-14T09:49:07,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/1d6b137e68aa4e06b69da6442719d232 is 50, key is test_row_0/B:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:07,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742150_1326 (size=12301) 2024-11-14T09:49:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:07,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577807905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T09:49:07,936 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:08,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/1d6b137e68aa4e06b69da6442719d232 2024-11-14T09:49:08,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8c5e7986d57b484c86bac6840907990d is 50, key is test_row_0/C:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:08,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742151_1327 (size=12301) 2024-11-14T09:49:08,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45584 deadline: 1731577808415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:08,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:08,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8c5e7986d57b484c86bac6840907990d 2024-11-14T09:49:08,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/32ec8defdbc247c99776d9e85f10af05 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05 2024-11-14T09:49:08,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05, entries=200, sequenceid=368, filesize=14.4 K 2024-11-14T09:49:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/1d6b137e68aa4e06b69da6442719d232 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232 2024-11-14T09:49:08,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232, entries=150, sequenceid=368, filesize=12.0 K 2024-11-14T09:49:08,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/8c5e7986d57b484c86bac6840907990d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d 2024-11-14T09:49:08,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d, entries=150, sequenceid=368, filesize=12.0 K 2024-11-14T09:49:08,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for ef41cd61287331f08dbf3f70d6eeab0f in 1314ms, sequenceid=368, compaction requested=true 2024-11-14T09:49:08,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:08,521 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ef41cd61287331f08dbf3f70d6eeab0f:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:08,521 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:08,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/A is initiating minor compaction (all files) 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/B is initiating minor compaction (all files) 2024-11-14T09:49:08,522 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/A in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:08,522 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/B in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:08,522 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32bf6f946d66445c8178ae4591f93e6f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=39.2 K 2024-11-14T09:49:08,522 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e1e4c4e203a04494a2e697dd74e71118, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.8 K 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32bf6f946d66445c8178ae4591f93e6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743684 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e1e4c4e203a04494a2e697dd74e71118, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743684 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b45ad84e604013a822f92f227d4f60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731577744382 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a60c898196547df8e9cc51adcfe1aa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731577744382 2024-11-14T09:49:08,522 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32ec8defdbc247c99776d9e85f10af05, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731577746564 2024-11-14T09:49:08,523 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d6b137e68aa4e06b69da6442719d232, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731577746564 2024-11-14T09:49:08,529 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:08,529 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9e750d1f0de4446a9f72064050aae18c is 50, key is test_row_0/A:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:08,529 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:08,530 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e566b5976f3a400baead4fa09be605d0 is 50, key is test_row_0/B:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:08,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742152_1328 (size=13221) 2024-11-14T09:49:08,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742153_1329 (size=13221) 2024-11-14T09:49:08,536 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/e566b5976f3a400baead4fa09be605d0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e566b5976f3a400baead4fa09be605d0 2024-11-14T09:49:08,539 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/B of ef41cd61287331f08dbf3f70d6eeab0f into e566b5976f3a400baead4fa09be605d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:08,539 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:08,540 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/B, priority=13, startTime=1731577748521; duration=0sec 2024-11-14T09:49:08,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:08,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:B 2024-11-14T09:49:08,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:08,541 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:08,541 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): ef41cd61287331f08dbf3f70d6eeab0f/C is initiating minor compaction (all files) 2024-11-14T09:49:08,541 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ef41cd61287331f08dbf3f70d6eeab0f/C in TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:08,541 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1774fa222931441f96bb8a9f373d7935, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp, totalSize=36.8 K 2024-11-14T09:49:08,541 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1774fa222931441f96bb8a9f373d7935, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731577743684 2024-11-14T09:49:08,541 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting dec0b9fc8d904b33b6c7d82f7d4327da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731577744382 2024-11-14T09:49:08,542 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c5e7986d57b484c86bac6840907990d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731577746564 2024-11-14T09:49:08,547 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ef41cd61287331f08dbf3f70d6eeab0f#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:08,547 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d1e33d4c593e4d7dba0fb169a88ae0cc is 50, key is test_row_0/C:col10/1731577746564/Put/seqid=0 2024-11-14T09:49:08,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742154_1330 (size=13221) 2024-11-14T09:49:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:08,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:08,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:08,596 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-11-14T09:49:08,596 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:08,596 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,596 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:08,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/e75760b6fe964b7d9393c78c29fee6ad is 50, key is test_row_0/A:col10/1731577747265/Put/seqid=0 2024-11-14T09:49:08,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742155_1331 (size=12301) 2024-11-14T09:49:08,617 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-14T09:49:08,617 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-14T09:49:08,617 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:08,617 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:08,617 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,617 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,617 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,617 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:08,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:08,648 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-11-14T09:49:08,648 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:08,648 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,648 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:08,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:08,940 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/9e750d1f0de4446a9f72064050aae18c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9e750d1f0de4446a9f72064050aae18c 2024-11-14T09:49:08,948 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/A of ef41cd61287331f08dbf3f70d6eeab0f into 9e750d1f0de4446a9f72064050aae18c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:08,948 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:08,949 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/A, priority=13, startTime=1731577748521; duration=0sec 2024-11-14T09:49:08,949 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:08,949 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:A 2024-11-14T09:49:08,956 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/d1e33d4c593e4d7dba0fb169a88ae0cc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d1e33d4c593e4d7dba0fb169a88ae0cc 2024-11-14T09:49:08,960 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ef41cd61287331f08dbf3f70d6eeab0f/C of ef41cd61287331f08dbf3f70d6eeab0f into d1e33d4c593e4d7dba0fb169a88ae0cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:08,960 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:08,960 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., storeName=ef41cd61287331f08dbf3f70d6eeab0f/C, priority=13, startTime=1731577748521; duration=0sec 2024-11-14T09:49:08,960 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:08,960 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ef41cd61287331f08dbf3f70d6eeab0f:C 2024-11-14T09:49:09,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/e75760b6fe964b7d9393c78c29fee6ad 2024-11-14T09:49:09,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/bc884f1404934424b2fa23e6e1a7aa90 is 50, key is test_row_0/B:col10/1731577747265/Put/seqid=0 2024-11-14T09:49:09,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742156_1332 (size=12301) 2024-11-14T09:49:09,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/bc884f1404934424b2fa23e6e1a7aa90 2024-11-14T09:49:09,427 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-14T09:49:09,427 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:09,427 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:09,427 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:09,427 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4494 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4498 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4394 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4545 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4423 2024-11-14T09:49:09,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:49:09,428 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:49:09,428 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:49:09,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:09,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:09,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:09,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:49:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:09,432 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577749431"}]},"ts":"1731577749431"} 2024-11-14T09:49:09,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/7d44595342844d1eaa819e28a4c7f0b7 is 50, key is test_row_0/C:col10/1731577747265/Put/seqid=0 2024-11-14T09:49:09,433 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:49:09,433 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:49:09,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:49:09,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, UNASSIGN}] 2024-11-14T09:49:09,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742157_1333 (size=12301) 2024-11-14T09:49:09,436 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, UNASSIGN 2024-11-14T09:49:09,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ef41cd61287331f08dbf3f70d6eeab0f, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:09,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, UNASSIGN because future has completed 2024-11-14T09:49:09,437 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:49:09,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:09,590 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:09,590 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:49:09,590 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing ef41cd61287331f08dbf3f70d6eeab0f, disabling compactions & flushes 2024-11-14T09:49:09,590 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:09,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/7d44595342844d1eaa819e28a4c7f0b7 2024-11-14T09:49:09,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/e75760b6fe964b7d9393c78c29fee6ad as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e75760b6fe964b7d9393c78c29fee6ad 2024-11-14T09:49:09,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e75760b6fe964b7d9393c78c29fee6ad, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:49:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/bc884f1404934424b2fa23e6e1a7aa90 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/bc884f1404934424b2fa23e6e1a7aa90 2024-11-14T09:49:09,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/bc884f1404934424b2fa23e6e1a7aa90, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:49:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/7d44595342844d1eaa819e28a4c7f0b7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/7d44595342844d1eaa819e28a4c7f0b7 2024-11-14T09:49:09,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/7d44595342844d1eaa819e28a4c7f0b7, entries=150, sequenceid=391, filesize=12.0 K 2024-11-14T09:49:09,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for ef41cd61287331f08dbf3f70d6eeab0f in 1256ms, sequenceid=391, compaction requested=false 2024-11-14T09:49:09,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ef41cd61287331f08dbf3f70d6eeab0f: 2024-11-14T09:49:09,852 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:09,852 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:09,852 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. after waiting 0 ms 2024-11-14T09:49:09,852 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:09,852 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing ef41cd61287331f08dbf3f70d6eeab0f 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T09:49:09,855 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1d4591617a884485b6a5b73c2cb188e8 is 50, key is test_row_0/A:col10/1731577749425/Put/seqid=0 2024-11-14T09:49:09,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742158_1334 (size=12301) 2024-11-14T09:49:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:10,259 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1d4591617a884485b6a5b73c2cb188e8 2024-11-14T09:49:10,265 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13e0cfbefd794a0eb654cb139e93f5b3 is 50, key is test_row_0/B:col10/1731577749425/Put/seqid=0 2024-11-14T09:49:10,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742159_1335 (size=12301) 2024-11-14T09:49:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:10,669 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13e0cfbefd794a0eb654cb139e93f5b3 2024-11-14T09:49:10,674 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/cfd3d92e56a449818fa09a691f3ec9e5 is 50, key is test_row_0/C:col10/1731577749425/Put/seqid=0 2024-11-14T09:49:10,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742160_1336 (size=12301) 2024-11-14T09:49:11,078 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/cfd3d92e56a449818fa09a691f3ec9e5 2024-11-14T09:49:11,082 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/A/1d4591617a884485b6a5b73c2cb188e8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1d4591617a884485b6a5b73c2cb188e8 2024-11-14T09:49:11,086 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1d4591617a884485b6a5b73c2cb188e8, entries=150, sequenceid=400, filesize=12.0 K 2024-11-14T09:49:11,087 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/B/13e0cfbefd794a0eb654cb139e93f5b3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13e0cfbefd794a0eb654cb139e93f5b3 2024-11-14T09:49:11,091 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13e0cfbefd794a0eb654cb139e93f5b3, entries=150, sequenceid=400, filesize=12.0 K 2024-11-14T09:49:11,092 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/.tmp/C/cfd3d92e56a449818fa09a691f3ec9e5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/cfd3d92e56a449818fa09a691f3ec9e5 2024-11-14T09:49:11,096 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/cfd3d92e56a449818fa09a691f3ec9e5, entries=150, sequenceid=400, filesize=12.0 K 2024-11-14T09:49:11,096 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ef41cd61287331f08dbf3f70d6eeab0f in 1244ms, sequenceid=400, compaction requested=true 2024-11-14T09:49:11,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/2aa1aa3629ba49cfac417f0e6531b96b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9ea084713dcb4247aa9aca0aa07a1fc9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e4d13288141c49b988567fd1bdf3aa6d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1be8acd258e3482d8e42f2391932b773, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/94a0f157d8fb49c3b41809487c171c88, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/0e91d3304f0049209da359d4148d67c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/fae734cd67e94611a1723be49b3a8da8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32bf6f946d66445c8178ae4591f93e6f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05] to archive 2024-11-14T09:49:11,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:11,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/d428da5468f64adc8cd450c5fb22fe1b 2024-11-14T09:49:11,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1b47544e2f5b474f9b2d02a3ed6a13a7 2024-11-14T09:49:11,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/2aa1aa3629ba49cfac417f0e6531b96b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/2aa1aa3629ba49cfac417f0e6531b96b 2024-11-14T09:49:11,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/edb3cf013901470ea635408e2186495b 2024-11-14T09:49:11,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f0123081776641aca955b5f6c41e1dd0 2024-11-14T09:49:11,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9ea084713dcb4247aa9aca0aa07a1fc9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9ea084713dcb4247aa9aca0aa07a1fc9 2024-11-14T09:49:11,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/959c6179297549228339e0ce0797c28d 2024-11-14T09:49:11,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c914b7b219624dbaa471bdc9e660a5e2 2024-11-14T09:49:11,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e4d13288141c49b988567fd1bdf3aa6d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e4d13288141c49b988567fd1bdf3aa6d 2024-11-14T09:49:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/a7f179986d8b40e08104a53ac580e1e3 2024-11-14T09:49:11,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/28d1344bab314284aa1527917b9d94bc 2024-11-14T09:49:11,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/bea1199794af46e9be45ef2fe219dc9e 2024-11-14T09:49:11,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1be8acd258e3482d8e42f2391932b773 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1be8acd258e3482d8e42f2391932b773 2024-11-14T09:49:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/f56e983c5f474fefbd1b1154390fd121 2024-11-14T09:49:11,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9b0d6d20f82d444e8b5a63d9715ff216 2024-11-14T09:49:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/94a0f157d8fb49c3b41809487c171c88 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/94a0f157d8fb49c3b41809487c171c88 2024-11-14T09:49:11,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/c5532690a69e49fc90e5fd23a3206aa4 2024-11-14T09:49:11,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/0e91d3304f0049209da359d4148d67c6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/0e91d3304f0049209da359d4148d67c6 2024-11-14T09:49:11,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/b679425e15ea4720ae2a7f01201ca32c 2024-11-14T09:49:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/5ce04684cd184075a190dfa1282d4be7 2024-11-14T09:49:11,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/fae734cd67e94611a1723be49b3a8da8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/fae734cd67e94611a1723be49b3a8da8 2024-11-14T09:49:11,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/03b6669ee3494100b885898879f30395 2024-11-14T09:49:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/409ceb0b015f4fbc8743c292d21ba595 2024-11-14T09:49:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/8a6d19fc4c49459881c4683ae069aec6 2024-11-14T09:49:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32bf6f946d66445c8178ae4591f93e6f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32bf6f946d66445c8178ae4591f93e6f 2024-11-14T09:49:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/15b45ad84e604013a822f92f227d4f60 2024-11-14T09:49:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/32ec8defdbc247c99776d9e85f10af05 2024-11-14T09:49:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/12ae953605e3469596ed61d83e16bd5a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14e3161361654d1c95fb695e5f304f91, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e8adca6e086649a1900a3d3cba07ec0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/faafdb6f65eb418299747ec28d08ae21, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/fc19db8cfeb4430e8ca7dfccfef2adc8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/2418f15741de4e69b80df71a0c3cbc51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ff56d4cd154426b8a8b9401a8f4af80, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e1e4c4e203a04494a2e697dd74e71118, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232] to archive 2024-11-14T09:49:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/275bf9b9b755410d9e1b9ce072ead46b 2024-11-14T09:49:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13a0357556624abca1292bb18ef0c698 2024-11-14T09:49:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/12ae953605e3469596ed61d83e16bd5a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/12ae953605e3469596ed61d83e16bd5a 2024-11-14T09:49:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/f5114ec3df1f47f7a6df010188ffaa0a 2024-11-14T09:49:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dbbf7a22c8514a6780af17acb5b8ad69 2024-11-14T09:49:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14e3161361654d1c95fb695e5f304f91 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14e3161361654d1c95fb695e5f304f91 2024-11-14T09:49:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ff28f735811e44bd87ee4aaaab5d3cd3 2024-11-14T09:49:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ed0ad7c3f864588bee4219811fcd981 2024-11-14T09:49:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e8adca6e086649a1900a3d3cba07ec0f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e8adca6e086649a1900a3d3cba07ec0f 2024-11-14T09:49:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/14ce33ba7020413e9ee5e2d4aaa17456 2024-11-14T09:49:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ed206e881fb14020aedb4ea1723f1338 2024-11-14T09:49:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/faafdb6f65eb418299747ec28d08ae21 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/faafdb6f65eb418299747ec28d08ae21 2024-11-14T09:49:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/d07024bb54de4c4091a106796d88e893 2024-11-14T09:49:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/dc6f29de9d524292bb168e0c89a5b471 2024-11-14T09:49:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/fc19db8cfeb4430e8ca7dfccfef2adc8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/fc19db8cfeb4430e8ca7dfccfef2adc8 2024-11-14T09:49:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce9681bf747f4d27b46e66469d380024 2024-11-14T09:49:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/46e93ece96c54cc088bbcbaebd5d2b60 2024-11-14T09:49:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/2418f15741de4e69b80df71a0c3cbc51 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/2418f15741de4e69b80df71a0c3cbc51 2024-11-14T09:49:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/66f95a801a3342ddab645c5fe93a052e 2024-11-14T09:49:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/196afc0ce749423782f6a6cf2667f5e1 2024-11-14T09:49:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ff56d4cd154426b8a8b9401a8f4af80 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/4ff56d4cd154426b8a8b9401a8f4af80 2024-11-14T09:49:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/ce2b57591a9a423680dd017fa4248e78 2024-11-14T09:49:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/b83c420c94e545a382ee1f46ce57fc09 2024-11-14T09:49:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e1e4c4e203a04494a2e697dd74e71118 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e1e4c4e203a04494a2e697dd74e71118 2024-11-14T09:49:11,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/8fc8096b146f45f3bc6437fc061b8abb 2024-11-14T09:49:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/7a60c898196547df8e9cc51adcfe1aa0 2024-11-14T09:49:11,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/1d6b137e68aa4e06b69da6442719d232 2024-11-14T09:49:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6dee8b4fcd7462fa6be889d0fcfa1f1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/373de8dff98a45b3951fe3eef899cc38, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b9241a408e1b4a0587f8c9ea80ddac2a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fe436bb98fc4853a2916e0120621693, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/0aeb42a7a9234de2ac6ee0d7269ddaf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6f81f09613c42f0b4a051b51b019505, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1774fa222931441f96bb8a9f373d7935, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d] to archive 2024-11-14T09:49:11,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:11,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1c34146dd50d48e5a6c4d315fbbb2309 2024-11-14T09:49:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d9102c6a42e443b7ae3fd72d80218232 2024-11-14T09:49:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6dee8b4fcd7462fa6be889d0fcfa1f1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6dee8b4fcd7462fa6be889d0fcfa1f1 2024-11-14T09:49:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/de5a159b13a74bf0990991509dce29d4 2024-11-14T09:49:11,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/bb89c77819bb48f3b013a4c9240ba9eb 2024-11-14T09:49:11,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/3b84ddb5ca2341f7bc7ac19fb5ed9b23 2024-11-14T09:49:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/99c3ec4b6465476ca12a49e8991fa37e 2024-11-14T09:49:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/aaff7d7c45e141cebb0b2311b7618673 2024-11-14T09:49:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/373de8dff98a45b3951fe3eef899cc38 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/373de8dff98a45b3951fe3eef899cc38 2024-11-14T09:49:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/22b89bfad6d14006baca8b292a3d78e5 2024-11-14T09:49:11,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/38495a16beb742dba5959cdff6574288 2024-11-14T09:49:11,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b9241a408e1b4a0587f8c9ea80ddac2a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b9241a408e1b4a0587f8c9ea80ddac2a 2024-11-14T09:49:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6a15c1b624b64d78a07632898c525ab3 2024-11-14T09:49:11,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/b2fe567167d74566bc59e93192cd0f85 2024-11-14T09:49:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fe436bb98fc4853a2916e0120621693 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fe436bb98fc4853a2916e0120621693 2024-11-14T09:49:11,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d405d83e63f64cb8be1aa61ad7a89bac 2024-11-14T09:49:11,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/2ecaab52a19b4a78bf6f8df7583c6579 2024-11-14T09:49:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/0aeb42a7a9234de2ac6ee0d7269ddaf8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/0aeb42a7a9234de2ac6ee0d7269ddaf8 2024-11-14T09:49:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6fd9b5fc549d4934b94605d84dd768c9 2024-11-14T09:49:11,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/6688fabf1ddf4967bd4d39c51459a9e8 2024-11-14T09:49:11,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6f81f09613c42f0b4a051b51b019505 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/e6f81f09613c42f0b4a051b51b019505 2024-11-14T09:49:11,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/196863a7780c415595fcc6532430d3e6 2024-11-14T09:49:11,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/257127e5b54741768864276db156730a 2024-11-14T09:49:11,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1774fa222931441f96bb8a9f373d7935 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/1774fa222931441f96bb8a9f373d7935 2024-11-14T09:49:11,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8e82354642724763bc2592b2facbca51 2024-11-14T09:49:11,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/dec0b9fc8d904b33b6c7d82f7d4327da 2024-11-14T09:49:11,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/8c5e7986d57b484c86bac6840907990d 2024-11-14T09:49:11,204 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/recovered.edits/403.seqid, newMaxSeqId=403, maxSeqId=1 2024-11-14T09:49:11,204 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. 2024-11-14T09:49:11,204 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for ef41cd61287331f08dbf3f70d6eeab0f: Waiting for close lock at 1731577749590Running coprocessor pre-close hooks at 1731577749590Disabling compacts and flushes for region at 1731577749590Disabling writes for close at 1731577749852 (+262 ms)Obtaining lock to block concurrent updates at 1731577749852Preparing flush snapshotting stores in ef41cd61287331f08dbf3f70d6eeab0f at 1731577749852Finished memstore snapshotting TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731577749852Flushing stores of TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f. at 1731577749853 (+1 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/A: creating writer at 1731577749853Flushing ef41cd61287331f08dbf3f70d6eeab0f/A: appending metadata at 1731577749855 (+2 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/A: closing flushed file at 1731577749855Flushing ef41cd61287331f08dbf3f70d6eeab0f/B: creating writer at 1731577750262 (+407 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/B: appending metadata at 1731577750265 (+3 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/B: closing flushed file at 1731577750265Flushing ef41cd61287331f08dbf3f70d6eeab0f/C: creating writer at 1731577750672 (+407 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/C: appending metadata at 1731577750674 (+2 ms)Flushing ef41cd61287331f08dbf3f70d6eeab0f/C: closing flushed file at 1731577750674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@745ef683: reopening flushed file at 1731577751081 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17136343: reopening flushed file at 1731577751086 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@590e6e6c: reopening flushed file at 1731577751091 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for ef41cd61287331f08dbf3f70d6eeab0f in 1244ms, sequenceid=400, compaction requested=true at 1731577751096 (+5 ms)Writing region close event to WAL at 1731577751201 (+105 ms)Running coprocessor post-close hooks at 1731577751204 (+3 ms)Closed at 1731577751204 2024-11-14T09:49:11,206 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:11,207 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=ef41cd61287331f08dbf3f70d6eeab0f, regionState=CLOSED 2024-11-14T09:49:11,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:11,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T09:49:11,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure ef41cd61287331f08dbf3f70d6eeab0f, server=defc576eb6b7,40243,1731577658084 in 1.7720 sec 2024-11-14T09:49:11,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-14T09:49:11,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ef41cd61287331f08dbf3f70d6eeab0f, UNASSIGN in 1.7750 sec 2024-11-14T09:49:11,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T09:49:11,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7800 sec 2024-11-14T09:49:11,216 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577751215"}]},"ts":"1731577751215"} 2024-11-14T09:49:11,217 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:49:11,217 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:49:11,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7890 sec 2024-11-14T09:49:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T09:49:11,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:11,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:49:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,587 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T09:49:11,588 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,590 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:11,593 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/recovered.edits] 2024-11-14T09:49:11,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1d4591617a884485b6a5b73c2cb188e8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/1d4591617a884485b6a5b73c2cb188e8 2024-11-14T09:49:11,598 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9e750d1f0de4446a9f72064050aae18c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/9e750d1f0de4446a9f72064050aae18c 2024-11-14T09:49:11,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e75760b6fe964b7d9393c78c29fee6ad to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/A/e75760b6fe964b7d9393c78c29fee6ad 2024-11-14T09:49:11,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13e0cfbefd794a0eb654cb139e93f5b3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/13e0cfbefd794a0eb654cb139e93f5b3 2024-11-14T09:49:11,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/bc884f1404934424b2fa23e6e1a7aa90 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/bc884f1404934424b2fa23e6e1a7aa90 2024-11-14T09:49:11,607 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e566b5976f3a400baead4fa09be605d0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/B/e566b5976f3a400baead4fa09be605d0 2024-11-14T09:49:11,611 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/7d44595342844d1eaa819e28a4c7f0b7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/7d44595342844d1eaa819e28a4c7f0b7 2024-11-14T09:49:11,613 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/cfd3d92e56a449818fa09a691f3ec9e5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/cfd3d92e56a449818fa09a691f3ec9e5 2024-11-14T09:49:11,615 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d1e33d4c593e4d7dba0fb169a88ae0cc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/C/d1e33d4c593e4d7dba0fb169a88ae0cc 2024-11-14T09:49:11,619 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/recovered.edits/403.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f/recovered.edits/403.seqid 2024-11-14T09:49:11,620 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/ef41cd61287331f08dbf3f70d6eeab0f 2024-11-14T09:49:11,620 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:49:11,624 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,630 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:49:11,632 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:49:11,634 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,634 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:49:11,634 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577751634"}]},"ts":"9223372036854775807"} 2024-11-14T09:49:11,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:49:11,636 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ef41cd61287331f08dbf3f70d6eeab0f, NAME => 'TestAcidGuarantees,,1731577725366.ef41cd61287331f08dbf3f70d6eeab0f.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:49:11,637 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:49:11,637 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577751637"}]},"ts":"9223372036854775807"} 2024-11-14T09:49:11,639 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:49:11,639 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-14T09:49:11,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T09:49:11,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:49:11,695 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:11,702 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1479602013_22 at /127.0.0.1:34764 [Waiting for operation #345] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-764308505_22 at /127.0.0.1:43216 [Waiting for operation #216] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=469 (was 416) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4188 (was 5123) 2024-11-14T09:49:11,710 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=469, ProcessCount=11, AvailableMemoryMB=4188 2024-11-14T09:49:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:49:11,711 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:49:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:11,714 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:49:11,714 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:11,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:49:11,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-14T09:49:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742161_1337 (size=959) 2024-11-14T09:49:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:12,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:49:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742162_1338 (size=53) 2024-11-14T09:49:12,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:12,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:49:12,528 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9e80b15335c1b44bab6a91f9a0dcea20, disabling compactions & flushes 2024-11-14T09:49:12,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. after waiting 0 ms 2024-11-14T09:49:12,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e80b15335c1b44bab6a91f9a0dcea20: Waiting for close lock at 1731577752528Disabling compacts and flushes for region at 1731577752528Disabling writes for close at 1731577752529 (+1 ms)Writing region close event to WAL at 1731577752529Closed at 1731577752529 2024-11-14T09:49:12,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:49:12,533 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577752532"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577752532"}]},"ts":"1731577752532"} 2024-11-14T09:49:12,536 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:49:12,537 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:49:12,538 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577752538"}]},"ts":"1731577752538"} 2024-11-14T09:49:12,539 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:49:12,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, ASSIGN}] 2024-11-14T09:49:12,541 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, ASSIGN 2024-11-14T09:49:12,542 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:49:12,692 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:12,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, ASSIGN because future has completed 2024-11-14T09:49:12,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:12,851 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,851 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:49:12,851 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,851 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:49:12,851 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,851 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,853 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:12,856 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName A 2024-11-14T09:49:12,856 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:12,856 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:12,857 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,858 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName B 2024-11-14T09:49:12,858 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:12,859 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:12,859 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,861 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName C 2024-11-14T09:49:12,861 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:12,862 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:12,863 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,863 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,864 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,865 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,865 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,865 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:49:12,867 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,868 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:49:12,869 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 9e80b15335c1b44bab6a91f9a0dcea20; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71913495, jitterRate=0.07159458100795746}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:49:12,869 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:12,870 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 9e80b15335c1b44bab6a91f9a0dcea20: Running coprocessor pre-open hook at 1731577752852Writing region info on filesystem at 1731577752852Initializing all the Stores at 1731577752853 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577752853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577752853Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577752853Cleaning up temporary data from old regions at 1731577752865 (+12 ms)Running coprocessor post-open hooks at 1731577752869 (+4 ms)Region opened successfully at 1731577752870 (+1 ms) 2024-11-14T09:49:12,870 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., pid=87, masterSystemTime=1731577752847 2024-11-14T09:49:12,872 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,872 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:12,873 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:12,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:12,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T09:49:12,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 in 180 msec 2024-11-14T09:49:12,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T09:49:12,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, ASSIGN in 338 msec 2024-11-14T09:49:12,880 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:49:12,881 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577752880"}]},"ts":"1731577752880"} 2024-11-14T09:49:12,883 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:49:12,884 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:49:12,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1730 sec 2024-11-14T09:49:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T09:49:13,866 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:13,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:13,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:13,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:13,868 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c36da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:13,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:13,870 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1854f22a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:13,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:13,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:13,873 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:49:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:49:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c21ed34 2024-11-14T09:49:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:49:13,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:49:13,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:49:13,876 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:49:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:13,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742163_1339 (size=995) 2024-11-14T09:49:14,289 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T09:49:14,289 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T09:49:14,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:49:14,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE}] 2024-11-14T09:49:14,297 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE 2024-11-14T09:49:14,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE because future has completed 2024-11-14T09:49:14,299 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:49:14,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:14,451 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,451 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:49:14,451 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 9e80b15335c1b44bab6a91f9a0dcea20, disabling compactions & flushes 2024-11-14T09:49:14,451 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,451 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,451 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. after waiting 0 ms 2024-11-14T09:49:14,451 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,456 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:49:14,457 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,457 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 9e80b15335c1b44bab6a91f9a0dcea20: Waiting for close lock at 1731577754451Running coprocessor pre-close hooks at 1731577754451Disabling compacts and flushes for region at 1731577754451Disabling writes for close at 1731577754451Writing region close event to WAL at 1731577754452 (+1 ms)Running coprocessor post-close hooks at 1731577754457 (+5 ms)Closed at 1731577754457 2024-11-14T09:49:14,457 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 9e80b15335c1b44bab6a91f9a0dcea20 to self. 2024-11-14T09:49:14,459 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,460 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=CLOSED 2024-11-14T09:49:14,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:14,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T09:49:14,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 in 165 msec 2024-11-14T09:49:14,468 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=true 2024-11-14T09:49:14,619 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE because future has completed 2024-11-14T09:49:14,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:14,775 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,775 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:49:14,776 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,776 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:49:14,776 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,776 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,777 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,778 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName A 2024-11-14T09:49:14,779 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:14,780 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:14,780 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,781 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName B 2024-11-14T09:49:14,781 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:14,782 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:14,782 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,783 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e80b15335c1b44bab6a91f9a0dcea20 columnFamilyName C 2024-11-14T09:49:14,783 DEBUG [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:14,783 INFO [StoreOpener-9e80b15335c1b44bab6a91f9a0dcea20-1 {}] regionserver.HStore(327): Store=9e80b15335c1b44bab6a91f9a0dcea20/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:14,783 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,784 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,786 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,787 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,787 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,788 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:49:14,790 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,791 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 9e80b15335c1b44bab6a91f9a0dcea20; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71043728, jitterRate=0.058634042739868164}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:49:14,791 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,792 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 9e80b15335c1b44bab6a91f9a0dcea20: Running coprocessor pre-open hook at 1731577754776Writing region info on filesystem at 1731577754776Initializing all the Stores at 1731577754777 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577754777Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577754777Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577754777Cleaning up temporary data from old regions at 1731577754787 (+10 ms)Running coprocessor post-open hooks at 1731577754791 (+4 ms)Region opened successfully at 1731577754792 (+1 ms) 2024-11-14T09:49:14,793 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., pid=92, masterSystemTime=1731577754772 2024-11-14T09:49:14,795 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,795 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,795 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:14,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-14T09:49:14,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 in 176 msec 2024-11-14T09:49:14,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T09:49:14,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, REOPEN/MOVE in 503 msec 2024-11-14T09:49:14,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T09:49:14,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-14T09:49:14,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-14T09:49:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T09:49:14,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:14,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8735c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321b7ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5753b40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62c17e4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c49e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,815 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ee333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,817 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa1c21d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,819 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44938577, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,821 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1891648d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161899fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38282c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abcea2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d074659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d879197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac9bfb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,831 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdeac14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,832 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f63200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,833 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749d2404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:14,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:14,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23450a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:14,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23aa1585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:14,837 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,837 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,837 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,837 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,837 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,838 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,838 DEBUG [Thread-1547 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,838 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,838 DEBUG [Thread-1547 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,838 DEBUG [Thread-1562 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,839 DEBUG [Thread-1562 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,839 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,839 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T09:49:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,840 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,840 DEBUG [Thread-1545 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,840 DEBUG [Thread-1545 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:14,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,843 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:14,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:14,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:49:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:14,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:14,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:49:14,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111442abd86a8df14bb7bca5c8f508e91140_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_1/A:col10/1731577754896/Put/seqid=0 2024-11-14T09:49:14,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:14,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577814930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577814929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:14,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742164_1340 (size=12154) 2024-11-14T09:49:14,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577814936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577814937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577814937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:14,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:14,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:14,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:14,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577815045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577815047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577815047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577815055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577815055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577815254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577815254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577815255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577815265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577815265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111442abd86a8df14bb7bca5c8f508e91140_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442abd86a8df14bb7bca5c8f508e91140_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:15,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/d7336a274b17420f89a0153e93715cf3, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:15,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/d7336a274b17420f89a0153e93715cf3 is 175, key is test_row_1/A:col10/1731577754896/Put/seqid=0 2024-11-14T09:49:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742165_1341 (size=30951) 2024-11-14T09:49:15,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577815564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577815564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577815564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577815575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577815575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:15,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/d7336a274b17420f89a0153e93715cf3 2024-11-14T09:49:15,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/69eefc8e4cd341c4930dd02e5aea32ff is 50, key is test_row_1/B:col10/1731577754896/Put/seqid=0 2024-11-14T09:49:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742166_1342 (size=9657) 2024-11-14T09:49:15,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/69eefc8e4cd341c4930dd02e5aea32ff 2024-11-14T09:49:15,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/63aab05b05e74f7fbd4847f25c5cbcef is 50, key is test_row_1/C:col10/1731577754896/Put/seqid=0 2024-11-14T09:49:15,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742167_1343 (size=9657) 2024-11-14T09:49:15,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:15,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:16,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:16,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:16,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577816074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577816085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:16,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577816085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577816095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577816095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:16,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:16,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:16,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:16,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/63aab05b05e74f7fbd4847f25c5cbcef 2024-11-14T09:49:16,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/d7336a274b17420f89a0153e93715cf3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3 2024-11-14T09:49:16,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3, entries=150, sequenceid=16, filesize=30.2 K 2024-11-14T09:49:16,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/69eefc8e4cd341c4930dd02e5aea32ff as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff 2024-11-14T09:49:16,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff, entries=100, sequenceid=16, filesize=9.4 K 2024-11-14T09:49:16,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/63aab05b05e74f7fbd4847f25c5cbcef as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef 2024-11-14T09:49:16,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef, entries=100, sequenceid=16, filesize=9.4 K 2024-11-14T09:49:16,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1397ms, sequenceid=16, compaction requested=false 2024-11-14T09:49:16,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:16,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T09:49:16,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:16,373 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:49:16,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144524fcc5ca6447989736d7fd5c1a338e_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577754924/Put/seqid=0 2024-11-14T09:49:16,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742168_1344 (size=12154) 2024-11-14T09:49:16,743 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:49:16,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:16,826 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144524fcc5ca6447989736d7fd5c1a338e_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144524fcc5ca6447989736d7fd5c1a338e_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/35603f3f8fa341fe867c904221efd760, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/35603f3f8fa341fe867c904221efd760 is 175, key is test_row_0/A:col10/1731577754924/Put/seqid=0 2024-11-14T09:49:16,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742169_1345 (size=30955) 2024-11-14T09:49:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:17,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577817135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577817136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577817137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577817138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577817138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,231 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/35603f3f8fa341fe867c904221efd760 2024-11-14T09:49:17,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3f18ed4e1e43414c8b4d8bb6f507637a is 50, key is test_row_0/B:col10/1731577754924/Put/seqid=0 2024-11-14T09:49:17,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742170_1346 (size=12001) 2024-11-14T09:49:17,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577817244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577817255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577817255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577817255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577817255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577817455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577817465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577817465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577817465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577817465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,644 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3f18ed4e1e43414c8b4d8bb6f507637a 2024-11-14T09:49:17,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/f525b0eaf3b14870aea9fea7792fb7f4 is 50, key is test_row_0/C:col10/1731577754924/Put/seqid=0 2024-11-14T09:49:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742171_1347 (size=12001) 2024-11-14T09:49:17,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577817775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577817775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577817775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577817775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:17,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577817775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,053 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/f525b0eaf3b14870aea9fea7792fb7f4 2024-11-14T09:49:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/35603f3f8fa341fe867c904221efd760 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760 2024-11-14T09:49:18,061 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760, entries=150, sequenceid=41, filesize=30.2 K 2024-11-14T09:49:18,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3f18ed4e1e43414c8b4d8bb6f507637a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a 2024-11-14T09:49:18,064 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T09:49:18,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/f525b0eaf3b14870aea9fea7792fb7f4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4 2024-11-14T09:49:18,068 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T09:49:18,069 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1696ms, sequenceid=41, compaction requested=false 2024-11-14T09:49:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T09:49:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T09:49:18,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T09:49:18,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2280 sec 2024-11-14T09:49:18,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.2360 sec 2024-11-14T09:49:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:18,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:49:18,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa9f03fa367b4fa2b44f1c440c53307a_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742172_1348 (size=17034) 2024-11-14T09:49:18,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577818342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577818343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577818344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577818347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577818348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577818455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577818455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577818455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577818455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577818465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577818665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577818665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577818665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577818665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577818675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,715 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:18,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fa9f03fa367b4fa2b44f1c440c53307a_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa9f03fa367b4fa2b44f1c440c53307a_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:18,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/39eac5ac99a649edaadb8b21b7391493, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:18,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/39eac5ac99a649edaadb8b21b7391493 is 175, key is test_row_0/A:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742173_1349 (size=48139) 2024-11-14T09:49:18,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577818975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577818975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577818975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577818975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577818984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T09:49:18,995 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:18,995 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T09:49:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:18,997 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:18,997 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:18,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:19,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/39eac5ac99a649edaadb8b21b7391493 2024-11-14T09:49:19,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/6040dd93e1034216a563457a1ac4a0e1 is 50, key is test_row_0/B:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:19,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742174_1350 (size=12001) 2024-11-14T09:49:19,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/6040dd93e1034216a563457a1ac4a0e1 2024-11-14T09:49:19,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/0c73b3069c524d89be6dc669d9d6bc3b is 50, key is test_row_0/C:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:19,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:49:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742175_1351 (size=12001) 2024-11-14T09:49:19,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:49:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:19,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:49:19,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:19,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577819485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577819485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577819485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:19,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:19,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577819494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:19,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577819504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:19,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:19,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/0c73b3069c524d89be6dc669d9d6bc3b 2024-11-14T09:49:19,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/39eac5ac99a649edaadb8b21b7391493 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493 2024-11-14T09:49:19,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493, entries=250, sequenceid=55, filesize=47.0 K 2024-11-14T09:49:19,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/6040dd93e1034216a563457a1ac4a0e1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1 2024-11-14T09:49:19,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T09:49:19,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/0c73b3069c524d89be6dc669d9d6bc3b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b 2024-11-14T09:49:19,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T09:49:19,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1286ms, sequenceid=55, compaction requested=true 2024-11-14T09:49:19,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:19,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:19,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:19,574 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:19,574 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,574 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,574 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=32.9 K 2024-11-14T09:49:19,574 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=107.5 K 2024-11-14T09:49:19,574 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493] 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 69eefc8e4cd341c4930dd02e5aea32ff, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577754895 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7336a274b17420f89a0153e93715cf3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577754847 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f18ed4e1e43414c8b4d8bb6f507637a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577754924 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35603f3f8fa341fe867c904221efd760, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577754924 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6040dd93e1034216a563457a1ac4a0e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:19,575 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39eac5ac99a649edaadb8b21b7391493, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:19,584 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:19,585 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:19,585 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/15d0540e3b6e4571838913aa2b5c6592 is 50, key is test_row_0/B:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:19,585 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114503a2af259634ccab384bd2657bb100d_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:19,588 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114503a2af259634ccab384bd2657bb100d_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:19,588 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114503a2af259634ccab384bd2657bb100d_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:19,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742176_1352 (size=12104) 2024-11-14T09:49:19,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742177_1353 (size=4469) 2024-11-14T09:49:19,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T09:49:19,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:19,608 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:19,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143acbfa77740f4c2a8205fb60ca4941e1_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577758347/Put/seqid=0 2024-11-14T09:49:19,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742178_1354 (size=12154) 2024-11-14T09:49:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:19,999 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:20,000 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f140705947294ab081d5c67829c24497 is 175, key is test_row_0/A:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:20,002 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/15d0540e3b6e4571838913aa2b5c6592 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/15d0540e3b6e4571838913aa2b5c6592 2024-11-14T09:49:20,007 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into 15d0540e3b6e4571838913aa2b5c6592(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:20,007 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:20,007 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=13, startTime=1731577759574; duration=0sec 2024-11-14T09:49:20,007 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:20,007 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:20,007 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:20,008 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:20,008 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:20,008 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:20,008 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=32.9 K 2024-11-14T09:49:20,008 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 63aab05b05e74f7fbd4847f25c5cbcef, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731577754895 2024-11-14T09:49:20,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f525b0eaf3b14870aea9fea7792fb7f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577754924 2024-11-14T09:49:20,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c73b3069c524d89be6dc669d9d6bc3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:20,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742179_1355 (size=31058) 2024-11-14T09:49:20,015 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f140705947294ab081d5c67829c24497 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497 2024-11-14T09:49:20,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:20,022 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into f140705947294ab081d5c67829c24497(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:20,022 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:20,022 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=13, startTime=1731577759573; duration=0sec 2024-11-14T09:49:20,022 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:20,022 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:20,024 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:20,025 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/da9c4747d428481e88eedd3819ae7cb7 is 50, key is test_row_0/C:col10/1731577758286/Put/seqid=0 2024-11-14T09:49:20,025 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143acbfa77740f4c2a8205fb60ca4941e1_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143acbfa77740f4c2a8205fb60ca4941e1_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:20,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/51ad6dc158094fca91898014f6814bf0, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:20,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/51ad6dc158094fca91898014f6814bf0 is 175, key is test_row_0/A:col10/1731577758347/Put/seqid=0 2024-11-14T09:49:20,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742180_1356 (size=12104) 2024-11-14T09:49:20,050 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/da9c4747d428481e88eedd3819ae7cb7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/da9c4747d428481e88eedd3819ae7cb7 2024-11-14T09:49:20,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742181_1357 (size=30955) 2024-11-14T09:49:20,055 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/51ad6dc158094fca91898014f6814bf0 2024-11-14T09:49:20,057 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into da9c4747d428481e88eedd3819ae7cb7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:20,057 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:20,057 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=13, startTime=1731577759574; duration=0sec 2024-11-14T09:49:20,057 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:20,057 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/b6416178d829425d95355ca33f025620 is 50, key is test_row_0/B:col10/1731577758347/Put/seqid=0 2024-11-14T09:49:20,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742182_1358 (size=12001) 2024-11-14T09:49:20,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:20,469 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/b6416178d829425d95355ca33f025620 2024-11-14T09:49:20,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/195f6ab9b080424696e7db175070bbcd is 50, key is test_row_0/C:col10/1731577758347/Put/seqid=0 2024-11-14T09:49:20,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742183_1359 (size=12001) 2024-11-14T09:49:20,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577820514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,522 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577820515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577820520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577820521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577820522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577820635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577820635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577820636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577820636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577820845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577820845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577820845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577820845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:20,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:20,881 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/195f6ab9b080424696e7db175070bbcd 2024-11-14T09:49:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/51ad6dc158094fca91898014f6814bf0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0 2024-11-14T09:49:20,890 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0, entries=150, sequenceid=77, filesize=30.2 K 2024-11-14T09:49:20,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/b6416178d829425d95355ca33f025620 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620 2024-11-14T09:49:20,894 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T09:49:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/195f6ab9b080424696e7db175070bbcd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd 2024-11-14T09:49:20,907 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T09:49:20,909 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1301ms, sequenceid=77, compaction requested=false 2024-11-14T09:49:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:20,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T09:49:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T09:49:20,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T09:49:20,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-14T09:49:20,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9180 sec 2024-11-14T09:49:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T09:49:21,145 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:21,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T09:49:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:21,147 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:21,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:21,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:21,157 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:21,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465e65a2ab06246389cc66d5ea24d6a2d_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:21,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742184_1360 (size=17034) 2024-11-14T09:49:21,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577821192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577821202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577821202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:21,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577821314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577821314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577821324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577821325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577821524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577821524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577821535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577821535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:21,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465e65a2ab06246389cc66d5ea24d6a2d_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465e65a2ab06246389cc66d5ea24d6a2d_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:21,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/4ad471b5b4964478acd4e1e9b810f891, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:21,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/4ad471b5b4964478acd4e1e9b810f891 is 175, key is test_row_0/A:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742185_1361 (size=48139) 2024-11-14T09:49:21,581 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/4ad471b5b4964478acd4e1e9b810f891 2024-11-14T09:49:21,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bf2881495d2d4757a96edbebf1ba7b30 is 50, key is test_row_0/B:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:21,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742186_1362 (size=12001) 2024-11-14T09:49:21,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:21,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577821835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577821835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577821844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577821855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:21,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:21,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:21,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bf2881495d2d4757a96edbebf1ba7b30 2024-11-14T09:49:21,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/a589a39cb0794e88b293aa5ca9bf8d62 is 50, key is test_row_0/C:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:22,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742187_1363 (size=12001) 2024-11-14T09:49:22,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:22,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577822344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577822355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577822355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577822365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:22,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:22,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:22,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/a589a39cb0794e88b293aa5ca9bf8d62 2024-11-14T09:49:22,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/4ad471b5b4964478acd4e1e9b810f891 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891 2024-11-14T09:49:22,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891, entries=250, sequenceid=96, filesize=47.0 K 2024-11-14T09:49:22,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bf2881495d2d4757a96edbebf1ba7b30 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30 2024-11-14T09:49:22,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T09:49:22,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/a589a39cb0794e88b293aa5ca9bf8d62 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62 2024-11-14T09:49:22,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T09:49:22,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1261ms, sequenceid=96, compaction requested=true 2024-11-14T09:49:22,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:22,419 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:22,419 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:22,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:22,419 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:22,419 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:22,419 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,420 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/15d0540e3b6e4571838913aa2b5c6592, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=35.3 K 2024-11-14T09:49:22,420 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 15d0540e3b6e4571838913aa2b5c6592, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:22,420 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b6416178d829425d95355ca33f025620, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577758341 2024-11-14T09:49:22,420 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2881495d2d4757a96edbebf1ba7b30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760520 2024-11-14T09:49:22,421 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:22,421 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:22,421 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,421 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=107.6 K 2024-11-14T09:49:22,421 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,421 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891] 2024-11-14T09:49:22,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f140705947294ab081d5c67829c24497, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:22,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51ad6dc158094fca91898014f6814bf0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577758341 2024-11-14T09:49:22,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad471b5b4964478acd4e1e9b810f891, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760514 2024-11-14T09:49:22,426 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:22,427 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:22,427 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/eccc1c89abb747f8b8c748c5c5a764d6 is 50, key is test_row_0/B:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:22,429 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114eaa69366bfea45449cb057f8583a6f5e_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:22,431 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114eaa69366bfea45449cb057f8583a6f5e_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:22,431 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eaa69366bfea45449cb057f8583a6f5e_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:22,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742188_1364 (size=12207) 2024-11-14T09:49:22,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742189_1365 (size=4469) 2024-11-14T09:49:22,446 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#313 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:22,446 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/ce59542a7cc04205b2792c3a8f213a11 is 175, key is test_row_0/A:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:22,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742190_1366 (size=31161) 2024-11-14T09:49:22,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T09:49:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,529 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T09:49:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142acd08abb1b94f128843db1101bc9c4a_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577761199/Put/seqid=0 2024-11-14T09:49:22,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742191_1367 (size=12154) 2024-11-14T09:49:22,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:22,542 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142acd08abb1b94f128843db1101bc9c4a_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acd08abb1b94f128843db1101bc9c4a_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:22,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/0305929676834c30933499e97478a4e2, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:22,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/0305929676834c30933499e97478a4e2 is 175, key is test_row_0/A:col10/1731577761199/Put/seqid=0 2024-11-14T09:49:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742192_1368 (size=30955) 2024-11-14T09:49:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:22,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:22,548 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/0305929676834c30933499e97478a4e2 2024-11-14T09:49:22,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/10c2ff8dd1f94adfbd37350fe5e2fe7b is 50, key is test_row_0/B:col10/1731577761199/Put/seqid=0 2024-11-14T09:49:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742193_1369 (size=12001) 2024-11-14T09:49:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577822614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577822725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,846 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/eccc1c89abb747f8b8c748c5c5a764d6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/eccc1c89abb747f8b8c748c5c5a764d6 2024-11-14T09:49:22,850 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into eccc1c89abb747f8b8c748c5c5a764d6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:22,850 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:22,850 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=13, startTime=1731577762419; duration=0sec 2024-11-14T09:49:22,850 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:22,850 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:22,850 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:22,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:22,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:22,851 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,851 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/da9c4747d428481e88eedd3819ae7cb7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=35.3 K 2024-11-14T09:49:22,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting da9c4747d428481e88eedd3819ae7cb7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731577757101 2024-11-14T09:49:22,851 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 195f6ab9b080424696e7db175070bbcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577758341 2024-11-14T09:49:22,852 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a589a39cb0794e88b293aa5ca9bf8d62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760520 2024-11-14T09:49:22,854 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/ce59542a7cc04205b2792c3a8f213a11 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11 2024-11-14T09:49:22,859 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:22,859 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/5b962ef89c654f158c1957e72f4ad7f3 is 50, key is test_row_0/C:col10/1731577761157/Put/seqid=0 2024-11-14T09:49:22,860 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into ce59542a7cc04205b2792c3a8f213a11(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:22,860 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:22,860 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=13, startTime=1731577762418; duration=0sec 2024-11-14T09:49:22,860 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:22,860 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742194_1370 (size=12207) 2024-11-14T09:49:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577822934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:22,960 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/10c2ff8dd1f94adfbd37350fe5e2fe7b 2024-11-14T09:49:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/797bb7eaa24a430f8fb9c7a53e21a1cd is 50, key is test_row_0/C:col10/1731577761199/Put/seqid=0 2024-11-14T09:49:22,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742195_1371 (size=12001) 2024-11-14T09:49:22,975 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/797bb7eaa24a430f8fb9c7a53e21a1cd 2024-11-14T09:49:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/0305929676834c30933499e97478a4e2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2 2024-11-14T09:49:22,984 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2, entries=150, sequenceid=116, filesize=30.2 K 2024-11-14T09:49:22,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/10c2ff8dd1f94adfbd37350fe5e2fe7b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b 2024-11-14T09:49:22,989 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:49:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/797bb7eaa24a430f8fb9c7a53e21a1cd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd 2024-11-14T09:49:22,993 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T09:49:22,995 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9e80b15335c1b44bab6a91f9a0dcea20 in 465ms, sequenceid=116, compaction requested=false 2024-11-14T09:49:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T09:49:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T09:49:23,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T09:49:23,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-11-14T09:49:23,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8540 sec 2024-11-14T09:49:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:23,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:23,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T09:49:23,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d3d7cea7d35f477c9fd6c9b01685fc37_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:23,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742196_1372 (size=12304) 2024-11-14T09:49:23,269 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/5b962ef89c654f158c1957e72f4ad7f3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5b962ef89c654f158c1957e72f4ad7f3 2024-11-14T09:49:23,273 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into 5b962ef89c654f158c1957e72f4ad7f3(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:23,273 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:23,273 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=13, startTime=1731577762419; duration=0sec 2024-11-14T09:49:23,273 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:23,273 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T09:49:23,295 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:23,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T09:49:23,296 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:23,297 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:23,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577823373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577823375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577823375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577823377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,386 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577823385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,387 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:23,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577823495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577823495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577823495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:23,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:23,665 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d3d7cea7d35f477c9fd6c9b01685fc37_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3d7cea7d35f477c9fd6c9b01685fc37_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:23,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f3ca17243b6d4a4fa225e80c211009c0, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:23,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f3ca17243b6d4a4fa225e80c211009c0 is 175, key is test_row_0/A:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:23,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742197_1373 (size=31105) 2024-11-14T09:49:23,677 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f3ca17243b6d4a4fa225e80c211009c0 2024-11-14T09:49:23,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/581f1855f7ec4909add136625a5a9911 is 50, key is test_row_0/B:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742198_1374 (size=12151) 2024-11-14T09:49:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577823705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577823705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577823705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:23,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:23,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:23,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:23,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577824015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577824015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577824015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/581f1855f7ec4909add136625a5a9911 2024-11-14T09:49:24,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/77f29657f93e4349b20233b4e7e9c502 is 50, key is test_row_0/C:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:24,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742199_1375 (size=12151) 2024-11-14T09:49:24,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:24,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/77f29657f93e4349b20233b4e7e9c502 2024-11-14T09:49:24,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f3ca17243b6d4a4fa225e80c211009c0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0 2024-11-14T09:49:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0, entries=150, sequenceid=135, filesize=30.4 K 2024-11-14T09:49:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/581f1855f7ec4909add136625a5a9911 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911 2024-11-14T09:49:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577824525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911, entries=150, sequenceid=135, filesize=11.9 K 2024-11-14T09:49:24,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/77f29657f93e4349b20233b4e7e9c502 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502 2024-11-14T09:49:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502, entries=150, sequenceid=135, filesize=11.9 K 2024-11-14T09:49:24,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1286ms, sequenceid=135, compaction requested=true 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:24,539 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:24,539 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:24,540 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,540 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,540 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/eccc1c89abb747f8b8c748c5c5a764d6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=35.5 K 2024-11-14T09:49:24,540 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=91.0 K 2024-11-14T09:49:24,540 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0] 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting eccc1c89abb747f8b8c748c5c5a764d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760520 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce59542a7cc04205b2792c3a8f213a11, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760520 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c2ff8dd1f94adfbd37350fe5e2fe7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577761191 2024-11-14T09:49:24,540 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0305929676834c30933499e97478a4e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577761191 2024-11-14T09:49:24,541 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 581f1855f7ec4909add136625a5a9911, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:24,541 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ca17243b6d4a4fa225e80c211009c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T09:49:24,548 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:24,549 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:24,550 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/01d4e1eac27b44b18e889e2e321c50ae is 50, key is test_row_0/B:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:24,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114597fdab411b14922bbbef9c8f78d939d_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577764542/Put/seqid=0 2024-11-14T09:49:24,555 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f535b01a739c40309799616bb68c5b6e_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:24,557 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f535b01a739c40309799616bb68c5b6e_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:24,557 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f535b01a739c40309799616bb68c5b6e_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742200_1376 (size=12459) 2024-11-14T09:49:24,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742201_1377 (size=14794) 2024-11-14T09:49:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742202_1378 (size=4469) 2024-11-14T09:49:24,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577824605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577824606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577824725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577824725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577824945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577824945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:24,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:24,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:24,973 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:24,974 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6f0a470941dd415da66ed062be241f80 is 175, key is test_row_0/A:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:24,979 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114597fdab411b14922bbbef9c8f78d939d_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114597fdab411b14922bbbef9c8f78d939d_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:24,980 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/3ddd8eb9b6d14ae4b16434f959d84fb9, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:24,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/3ddd8eb9b6d14ae4b16434f959d84fb9 is 175, key is test_row_0/A:col10/1731577764542/Put/seqid=0 2024-11-14T09:49:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,981 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/01d4e1eac27b44b18e889e2e321c50ae as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/01d4e1eac27b44b18e889e2e321c50ae 2024-11-14T09:49:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:24,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742203_1379 (size=31413) 2024-11-14T09:49:24,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742204_1380 (size=39749) 2024-11-14T09:49:24,989 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into 01d4e1eac27b44b18e889e2e321c50ae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:24,989 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:24,989 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=13, startTime=1731577764539; duration=0sec 2024-11-14T09:49:24,989 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:24,989 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:24,989 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:24,994 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:24,994 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:24,995 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:24,995 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5b962ef89c654f158c1957e72f4ad7f3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=35.5 K 2024-11-14T09:49:24,995 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b962ef89c654f158c1957e72f4ad7f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731577760520 2024-11-14T09:49:24,995 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 797bb7eaa24a430f8fb9c7a53e21a1cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731577761191 2024-11-14T09:49:24,996 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f29657f93e4349b20233b4e7e9c502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:25,003 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:25,003 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/6b268403a8c54b72b88d4faecdbfe7c1 is 50, key is test_row_0/C:col10/1731577763252/Put/seqid=0 2024-11-14T09:49:25,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742205_1381 (size=12459) 2024-11-14T09:49:25,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577825255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577825265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6f0a470941dd415da66ed062be241f80 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80 2024-11-14T09:49:25,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/3ddd8eb9b6d14ae4b16434f959d84fb9 2024-11-14T09:49:25,391 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into 6f0a470941dd415da66ed062be241f80(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:25,391 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:25,392 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=13, startTime=1731577764539; duration=0sec 2024-11-14T09:49:25,392 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:25,392 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:25,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/41f427c7025f493c8b540797e069bbf8 is 50, key is test_row_0/B:col10/1731577764542/Put/seqid=0 2024-11-14T09:49:25,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577825405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577825405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,410 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,410 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742206_1382 (size=12151) 2024-11-14T09:49:25,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/41f427c7025f493c8b540797e069bbf8 2024-11-14T09:49:25,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/40f060cd704c401ab378a2df36d2ead8 is 50, key is test_row_0/C:col10/1731577764542/Put/seqid=0 2024-11-14T09:49:25,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742207_1383 (size=12151) 2024-11-14T09:49:25,433 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/6b268403a8c54b72b88d4faecdbfe7c1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/6b268403a8c54b72b88d4faecdbfe7c1 2024-11-14T09:49:25,439 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into 6b268403a8c54b72b88d4faecdbfe7c1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:25,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:25,439 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=13, startTime=1731577764539; duration=0sec 2024-11-14T09:49:25,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:25,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:25,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:25,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577825545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,551 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577825764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577825775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:25,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/40f060cd704c401ab378a2df36d2ead8 2024-11-14T09:49:25,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/3ddd8eb9b6d14ae4b16434f959d84fb9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9 2024-11-14T09:49:25,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9, entries=200, sequenceid=156, filesize=38.8 K 2024-11-14T09:49:25,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/41f427c7025f493c8b540797e069bbf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8 2024-11-14T09:49:25,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:49:25,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/40f060cd704c401ab378a2df36d2ead8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8 2024-11-14T09:49:25,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T09:49:25,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1315ms, sequenceid=156, compaction requested=false 2024-11-14T09:49:25,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:25,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T09:49:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:25,900 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:49:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f14443a4d8464dada34f3946edad3bb5_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577764572/Put/seqid=0 2024-11-14T09:49:25,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742208_1384 (size=12304) 2024-11-14T09:49:26,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:26,317 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f14443a4d8464dada34f3946edad3bb5_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f14443a4d8464dada34f3946edad3bb5_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8aedcf9180d042389ae46f8d86552c1a, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8aedcf9180d042389ae46f8d86552c1a is 175, key is test_row_0/A:col10/1731577764572/Put/seqid=0 2024-11-14T09:49:26,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742209_1385 (size=31105) 2024-11-14T09:49:26,730 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8aedcf9180d042389ae46f8d86552c1a 2024-11-14T09:49:26,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/990ed907338d46b38d37b802ae4fa0fd is 50, key is test_row_0/B:col10/1731577764572/Put/seqid=0 2024-11-14T09:49:26,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742210_1386 (size=12151) 2024-11-14T09:49:26,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577826855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:26,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577826859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577826974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577826974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,140 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/990ed907338d46b38d37b802ae4fa0fd 2024-11-14T09:49:27,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4bb7aff0bb7c4cc2b51231b380a7a4cc is 50, key is test_row_0/C:col10/1731577764572/Put/seqid=0 2024-11-14T09:49:27,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742211_1387 (size=12151) 2024-11-14T09:49:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577827185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577827185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:27,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577827505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577827505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,550 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4bb7aff0bb7c4cc2b51231b380a7a4cc 2024-11-14T09:49:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8aedcf9180d042389ae46f8d86552c1a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a 2024-11-14T09:49:27,556 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a, entries=150, sequenceid=175, filesize=30.4 K 2024-11-14T09:49:27,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/990ed907338d46b38d37b802ae4fa0fd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd 2024-11-14T09:49:27,565 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd, entries=150, sequenceid=175, filesize=11.9 K 2024-11-14T09:49:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4bb7aff0bb7c4cc2b51231b380a7a4cc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc 2024-11-14T09:49:27,572 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc, entries=150, sequenceid=175, filesize=11.9 K 2024-11-14T09:49:27,573 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1674ms, sequenceid=175, compaction requested=true 2024-11-14T09:49:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:27,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T09:49:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T09:49:27,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T09:49:27,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.2780 sec 2024-11-14T09:49:27,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 4.2820 sec 2024-11-14T09:49:27,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:27,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:49:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:27,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114389488f44e024b839122111092cd01d5_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:27,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742212_1388 (size=14794) 2024-11-14T09:49:27,619 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:27,626 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114389488f44e024b839122111092cd01d5_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114389488f44e024b839122111092cd01d5_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:27,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e3f3078e65ca47e09a353e166cc62daa, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:27,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e3f3078e65ca47e09a353e166cc62daa is 175, key is test_row_0/A:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742213_1389 (size=39749) 2024-11-14T09:49:27,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577827702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:27,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577827825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:27,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577828014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:28,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577828014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:28,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e3f3078e65ca47e09a353e166cc62daa 2024-11-14T09:49:28,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/1571381210d6494f8b8f68b358a7645e is 50, key is test_row_0/B:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:28,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742214_1390 (size=12151) 2024-11-14T09:49:28,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577828045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:28,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577828354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:28,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:28,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/1571381210d6494f8b8f68b358a7645e 2024-11-14T09:49:28,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/c013c17893074cb486d859790ef4165d is 50, key is test_row_0/C:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:28,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742215_1391 (size=12151) 2024-11-14T09:49:28,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/c013c17893074cb486d859790ef4165d 2024-11-14T09:49:28,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e3f3078e65ca47e09a353e166cc62daa as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa 2024-11-14T09:49:28,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa, entries=200, sequenceid=196, filesize=38.8 K 2024-11-14T09:49:28,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/1571381210d6494f8b8f68b358a7645e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e 2024-11-14T09:49:28,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T09:49:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/c013c17893074cb486d859790ef4165d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d 2024-11-14T09:49:28,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d, entries=150, sequenceid=196, filesize=11.9 K 2024-11-14T09:49:28,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9e80b15335c1b44bab6a91f9a0dcea20 in 888ms, sequenceid=196, compaction requested=true 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:28,469 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:28,469 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:28,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:28,471 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:28,471 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:28,471 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:28,471 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:28,471 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:28,471 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:28,471 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/01d4e1eac27b44b18e889e2e321c50ae, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=47.8 K 2024-11-14T09:49:28,471 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=138.7 K 2024-11-14T09:49:28,471 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa] 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d4e1eac27b44b18e889e2e321c50ae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0a470941dd415da66ed062be241f80, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 41f427c7025f493c8b540797e069bbf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577763358 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ddd8eb9b6d14ae4b16434f959d84fb9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577763358 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 990ed907338d46b38d37b802ae4fa0fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731577764572 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aedcf9180d042389ae46f8d86552c1a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731577764572 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1571381210d6494f8b8f68b358a7645e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:28,472 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3f3078e65ca47e09a353e166cc62daa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:28,480 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:28,483 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:28,484 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/dae5813865b44be2b01c9f0341b2797a is 50, key is test_row_0/B:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:28,486 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a8a7955f5a794c18acdd674d4c9c3e21_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:28,489 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a8a7955f5a794c18acdd674d4c9c3e21_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:28,489 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a8a7955f5a794c18acdd674d4c9c3e21_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:28,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742216_1392 (size=12595) 2024-11-14T09:49:28,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742217_1393 (size=4469) 2024-11-14T09:49:28,502 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#333 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:28,503 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f5fb5cf645704c26906e161e26c4e74e is 175, key is test_row_0/A:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:28,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742218_1394 (size=31549) 2024-11-14T09:49:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:28,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:28,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:49:28,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e437a35cf3804fac90fc86265a10bff0_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577767683/Put/seqid=0 2024-11-14T09:49:28,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742219_1395 (size=14794) 2024-11-14T09:49:28,895 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/dae5813865b44be2b01c9f0341b2797a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/dae5813865b44be2b01c9f0341b2797a 2024-11-14T09:49:28,898 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into dae5813865b44be2b01c9f0341b2797a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:28,898 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:28,898 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=12, startTime=1731577768469; duration=0sec 2024-11-14T09:49:28,898 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:28,898 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:28,898 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:28,899 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:28,899 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:28,899 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:28,899 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/6b268403a8c54b72b88d4faecdbfe7c1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=47.8 K 2024-11-14T09:49:28,900 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b268403a8c54b72b88d4faecdbfe7c1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731577762597 2024-11-14T09:49:28,900 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f060cd704c401ab378a2df36d2ead8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731577763358 2024-11-14T09:49:28,900 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb7aff0bb7c4cc2b51231b380a7a4cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731577764572 2024-11-14T09:49:28,900 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c013c17893074cb486d859790ef4165d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:28,907 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:28,907 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/5bd7f9843e814143b690ce34bde9754a is 50, key is test_row_0/C:col10/1731577767579/Put/seqid=0 2024-11-14T09:49:28,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742220_1396 (size=12595) 2024-11-14T09:49:28,914 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/f5fb5cf645704c26906e161e26c4e74e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e 2024-11-14T09:49:28,916 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/5bd7f9843e814143b690ce34bde9754a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5bd7f9843e814143b690ce34bde9754a 2024-11-14T09:49:28,918 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into f5fb5cf645704c26906e161e26c4e74e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:28,918 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:28,918 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=12, startTime=1731577768469; duration=0sec 2024-11-14T09:49:28,919 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:28,919 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:28,928 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into 5bd7f9843e814143b690ce34bde9754a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:28,928 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:28,928 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=12, startTime=1731577768469; duration=0sec 2024-11-14T09:49:28,928 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:28,928 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577828999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577829035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577829035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,041 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:29,041 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577829115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:29,294 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e437a35cf3804fac90fc86265a10bff0_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e437a35cf3804fac90fc86265a10bff0_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:29,295 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e14d40170dc8461d90d290dd7c26ffd6, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:29,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e14d40170dc8461d90d290dd7c26ffd6 is 175, key is test_row_0/A:col10/1731577767683/Put/seqid=0 2024-11-14T09:49:29,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742221_1397 (size=39749) 2024-11-14T09:49:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577829324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43570 deadline: 1731577829434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,438 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43592 deadline: 1731577829444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,448 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577829645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:29,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:29,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e14d40170dc8461d90d290dd7c26ffd6 2024-11-14T09:49:29,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/c128f4971a094e1ca2fa0d8c85744f58 is 50, key is test_row_0/B:col10/1731577767683/Put/seqid=0 2024-11-14T09:49:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742222_1398 (size=12151) 2024-11-14T09:49:30,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/c128f4971a094e1ca2fa0d8c85744f58 2024-11-14T09:49:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577830155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:30,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4c0633dc32bc40e6a9c57fdebf68d646 is 50, key is test_row_0/C:col10/1731577767683/Put/seqid=0 2024-11-14T09:49:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742223_1399 (size=12151) 2024-11-14T09:49:30,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4c0633dc32bc40e6a9c57fdebf68d646 2024-11-14T09:49:30,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/e14d40170dc8461d90d290dd7c26ffd6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6 2024-11-14T09:49:30,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6, entries=200, sequenceid=212, filesize=38.8 K 2024-11-14T09:49:30,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/c128f4971a094e1ca2fa0d8c85744f58 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58 2024-11-14T09:49:30,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T09:49:30,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/4c0633dc32bc40e6a9c57fdebf68d646 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646 2024-11-14T09:49:30,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646, entries=150, sequenceid=212, filesize=11.9 K 2024-11-14T09:49:30,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1698ms, sequenceid=212, compaction requested=false 2024-11-14T09:49:30,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:31,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:31,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:49:31,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149239152f32d644fbbe4b70c0b22cccef_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742224_1400 (size=14794) 2024-11-14T09:49:31,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577831152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577831152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577831165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,169 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577831265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577831265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,478 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:31,485 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149239152f32d644fbbe4b70c0b22cccef_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149239152f32d644fbbe4b70c0b22cccef_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:31,486 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/dbd0923709da40cd8e1b224780553d49, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:31,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/dbd0923709da40cd8e1b224780553d49 is 175, key is test_row_0/A:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577831484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577831484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T09:49:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,495 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:31,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T09:49:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:31,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742225_1401 (size=39749) 2024-11-14T09:49:31,500 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:31,503 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:31,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:31,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/dbd0923709da40cd8e1b224780553d49 2024-11-14T09:49:31,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3caad0d3e21a426fa9c02d5daeeb6363 is 50, key is test_row_0/B:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742226_1402 (size=12151) 2024-11-14T09:49:31,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3caad0d3e21a426fa9c02d5daeeb6363 2024-11-14T09:49:31,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ab88425667444684a66de1b0a1e6c585 is 50, key is test_row_0/C:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742227_1403 (size=12151) 2024-11-14T09:49:31,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ab88425667444684a66de1b0a1e6c585 2024-11-14T09:49:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:31,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/dbd0923709da40cd8e1b224780553d49 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49 2024-11-14T09:49:31,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49, entries=200, sequenceid=236, filesize=38.8 K 2024-11-14T09:49:31,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/3caad0d3e21a426fa9c02d5daeeb6363 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363 2024-11-14T09:49:31,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363, entries=150, sequenceid=236, filesize=11.9 K 2024-11-14T09:49:31,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ab88425667444684a66de1b0a1e6c585 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585 2024-11-14T09:49:31,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585, entries=150, sequenceid=236, filesize=11.9 K 2024-11-14T09:49:31,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9e80b15335c1b44bab6a91f9a0dcea20 in 578ms, sequenceid=236, compaction requested=true 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:31,639 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:49:31,639 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:31,641 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:31,641 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:31,641 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=108.4 K 2024-11-14T09:49:31,641 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/dae5813865b44be2b01c9f0341b2797a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=36.0 K 2024-11-14T09:49:31,641 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49] 2024-11-14T09:49:31,641 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae5813865b44be2b01c9f0341b2797a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:31,642 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fb5cf645704c26906e161e26c4e74e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:31,642 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c128f4971a094e1ca2fa0d8c85744f58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731577767683 2024-11-14T09:49:31,642 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e14d40170dc8461d90d290dd7c26ffd6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731577767683 2024-11-14T09:49:31,642 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd0923709da40cd8e1b224780553d49, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:31,642 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3caad0d3e21a426fa9c02d5daeeb6363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:31,649 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:31,651 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149bf11d305936429cae629bcd576fffcf_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:31,651 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:31,651 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/404ae03e5c9e4ff3b5f9972eab3d4e6c is 50, key is test_row_0/B:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,653 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149bf11d305936429cae629bcd576fffcf_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:31,653 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149bf11d305936429cae629bcd576fffcf_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:31,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T09:49:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:31,656 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:31,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742228_1404 (size=12697) 2024-11-14T09:49:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742229_1405 (size=4469) 2024-11-14T09:49:31,689 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#342 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:31,689 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/404ae03e5c9e4ff3b5f9972eab3d4e6c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/404ae03e5c9e4ff3b5f9972eab3d4e6c 2024-11-14T09:49:31,690 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8bf96f05dd634ff2be7bedb56b65f5c4 is 175, key is test_row_0/A:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,696 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into 404ae03e5c9e4ff3b5f9972eab3d4e6c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:31,696 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:31,696 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=13, startTime=1731577771639; duration=0sec 2024-11-14T09:49:31,696 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:31,696 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:31,696 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:31,697 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:31,697 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:31,697 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:31,698 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5bd7f9843e814143b690ce34bde9754a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=36.0 K 2024-11-14T09:49:31,698 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd7f9843e814143b690ce34bde9754a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731577766833 2024-11-14T09:49:31,698 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0633dc32bc40e6a9c57fdebf68d646, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731577767683 2024-11-14T09:49:31,698 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab88425667444684a66de1b0a1e6c585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149e69f19fd7934ad29fc231f66267d4f1_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577771097/Put/seqid=0 2024-11-14T09:49:31,713 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:31,713 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/cdb70c6fff1341daa8ebf27b30dc077f is 50, key is test_row_0/C:col10/1731577771059/Put/seqid=0 2024-11-14T09:49:31,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742230_1406 (size=31651) 2024-11-14T09:49:31,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742231_1407 (size=12304) 2024-11-14T09:49:31,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742232_1408 (size=12697) 2024-11-14T09:49:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:31,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:31,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577831862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577831864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577831975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:31,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577831975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:31,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:32,121 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/8bf96f05dd634ff2be7bedb56b65f5c4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4 2024-11-14T09:49:32,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:32,125 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into 8bf96f05dd634ff2be7bedb56b65f5c4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:32,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:32,125 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=13, startTime=1731577771639; duration=0sec 2024-11-14T09:49:32,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:32,125 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:32,127 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149e69f19fd7934ad29fc231f66267d4f1_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149e69f19fd7934ad29fc231f66267d4f1_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:32,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6364b3f216d44597b62e3b3b55bc9aaa, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:32,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6364b3f216d44597b62e3b3b55bc9aaa is 175, key is test_row_0/A:col10/1731577771097/Put/seqid=0 2024-11-14T09:49:32,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742233_1409 (size=31105) 2024-11-14T09:49:32,132 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6364b3f216d44597b62e3b3b55bc9aaa 2024-11-14T09:49:32,133 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/cdb70c6fff1341daa8ebf27b30dc077f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/cdb70c6fff1341daa8ebf27b30dc077f 2024-11-14T09:49:32,137 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into cdb70c6fff1341daa8ebf27b30dc077f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:32,137 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:32,137 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=13, startTime=1731577771639; duration=0sec 2024-11-14T09:49:32,137 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:32,137 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/058753070f0f42bc8bbf849cf48c50e0 is 50, key is test_row_0/B:col10/1731577771097/Put/seqid=0 2024-11-14T09:49:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742234_1410 (size=12151) 2024-11-14T09:49:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577832185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577832185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:32,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577832495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:32,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577832504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:32,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:32,543 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/058753070f0f42bc8bbf849cf48c50e0 2024-11-14T09:49:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/1b05506e3b1446ababced466fd13f085 is 50, key is test_row_0/C:col10/1731577771097/Put/seqid=0 2024-11-14T09:49:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742235_1411 (size=12151) 2024-11-14T09:49:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:32,953 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/1b05506e3b1446ababced466fd13f085 2024-11-14T09:49:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/6364b3f216d44597b62e3b3b55bc9aaa as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa 2024-11-14T09:49:32,961 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa, entries=150, sequenceid=251, filesize=30.4 K 2024-11-14T09:49:32,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/058753070f0f42bc8bbf849cf48c50e0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0 2024-11-14T09:49:32,965 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:49:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/1b05506e3b1446ababced466fd13f085 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085 2024-11-14T09:49:32,969 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T09:49:32,970 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1314ms, sequenceid=251, compaction requested=false 2024-11-14T09:49:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T09:49:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T09:49:32,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T09:49:32,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-14T09:49:32,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4770 sec 2024-11-14T09:49:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:33,012 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:33,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:33,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111427ad65c5fc3a4c0fb64ef5a1025952d3_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:33,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742236_1412 (size=12454) 2024-11-14T09:49:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577833062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577833063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43584 deadline: 1731577833174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577833174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,180 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 9e80b15335c1b44bab6a91f9a0dcea20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577833185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577833385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577833395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,434 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:33,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111427ad65c5fc3a4c0fb64ef5a1025952d3_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111427ad65c5fc3a4c0fb64ef5a1025952d3_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:33,438 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/bf1c48d2faa849f3af9397b5e262ea26, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:33,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/bf1c48d2faa849f3af9397b5e262ea26 is 175, key is test_row_0/A:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:33,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742237_1413 (size=31255) 2024-11-14T09:49:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T09:49:33,645 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:33,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T09:49:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:33,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:33,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:33,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577833705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577833715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:33,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:33,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:33,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/bf1c48d2faa849f3af9397b5e262ea26 2024-11-14T09:49:33,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/0ba27f8912b9406b85b1b065ad8c147f is 50, key is test_row_0/B:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:33,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742238_1414 (size=12301) 2024-11-14T09:49:33,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:33,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:34,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43542 deadline: 1731577834214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:34,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:34,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43572 deadline: 1731577834224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:34,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:49:34,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/0ba27f8912b9406b85b1b065ad8c147f 2024-11-14T09:49:34,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/93b9ad9eff434cd88f1b588799babae7 is 50, key is test_row_0/C:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:34,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742239_1415 (size=12301) 2024-11-14T09:49:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:34,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:34,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/93b9ad9eff434cd88f1b588799babae7 2024-11-14T09:49:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/bf1c48d2faa849f3af9397b5e262ea26 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26 2024-11-14T09:49:34,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26, entries=150, sequenceid=276, filesize=30.5 K 2024-11-14T09:49:34,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/0ba27f8912b9406b85b1b065ad8c147f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f 2024-11-14T09:49:34,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f, entries=150, sequenceid=276, filesize=12.0 K 2024-11-14T09:49:34,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/93b9ad9eff434cd88f1b588799babae7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7 2024-11-14T09:49:34,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7, entries=150, sequenceid=276, filesize=12.0 K 2024-11-14T09:49:34,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1667ms, sequenceid=276, compaction requested=true 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:34,680 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:34,680 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9e80b15335c1b44bab6a91f9a0dcea20:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:34,680 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:34,680 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/A is initiating minor compaction (all files) 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/B is initiating minor compaction (all files) 2024-11-14T09:49:34,681 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/B in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,681 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/A in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,681 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=91.8 K 2024-11-14T09:49:34,681 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/404ae03e5c9e4ff3b5f9972eab3d4e6c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=36.3 K 2024-11-14T09:49:34,681 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26] 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 404ae03e5c9e4ff3b5f9972eab3d4e6c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bf96f05dd634ff2be7bedb56b65f5c4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 058753070f0f42bc8bbf849cf48c50e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577771097 2024-11-14T09:49:34,681 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ba27f8912b9406b85b1b065ad8c147f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577771851 2024-11-14T09:49:34,682 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6364b3f216d44597b62e3b3b55bc9aaa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577771097 2024-11-14T09:49:34,682 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1c48d2faa849f3af9397b5e262ea26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577771851 2024-11-14T09:49:34,690 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:34,690 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/6e77fb8986ef430e8d2c9e7631355412 is 50, key is test_row_0/B:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:34,690 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:34,693 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114138ed3dd20d24b96a203880e1e136f99_9e80b15335c1b44bab6a91f9a0dcea20 store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:34,695 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114138ed3dd20d24b96a203880e1e136f99_9e80b15335c1b44bab6a91f9a0dcea20, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:34,695 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114138ed3dd20d24b96a203880e1e136f99_9e80b15335c1b44bab6a91f9a0dcea20 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:34,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742240_1416 (size=12949) 2024-11-14T09:49:34,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742241_1417 (size=4469) 2024-11-14T09:49:34,700 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#A#compaction#352 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:34,701 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/73df2b818a5e4216bf1e30a475cf2f76 is 175, key is test_row_0/A:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:34,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742242_1418 (size=31903) 2024-11-14T09:49:34,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T09:49:34,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:34,720 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:49:34,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146ae4f2267b81436fa1d513c01de5cbb8_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577773020/Put/seqid=0 2024-11-14T09:49:34,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742243_1419 (size=12454) 2024-11-14T09:49:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:34,840 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-14T09:49:34,840 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:34,840 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,840 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:34,841 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-14T09:49:34,841 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:34,841 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,842 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:34,845 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-14T09:49:34,845 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:34,845 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,845 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:34,846 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-14T09:49:34,846 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:34,846 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,846 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:34,847 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-14T09:49:34,847 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:34,847 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,847 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:34,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:35,104 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/6e77fb8986ef430e8d2c9e7631355412 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6e77fb8986ef430e8d2c9e7631355412 2024-11-14T09:49:35,109 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/73df2b818a5e4216bf1e30a475cf2f76 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/73df2b818a5e4216bf1e30a475cf2f76 2024-11-14T09:49:35,109 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/B of 9e80b15335c1b44bab6a91f9a0dcea20 into 6e77fb8986ef430e8d2c9e7631355412(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:35,109 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:35,109 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/B, priority=13, startTime=1731577774680; duration=0sec 2024-11-14T09:49:35,110 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:35,110 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:B 2024-11-14T09:49:35,110 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:35,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:35,111 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 9e80b15335c1b44bab6a91f9a0dcea20/C is initiating minor compaction (all files) 2024-11-14T09:49:35,111 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9e80b15335c1b44bab6a91f9a0dcea20/C in TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:35,111 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/cdb70c6fff1341daa8ebf27b30dc077f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp, totalSize=36.3 K 2024-11-14T09:49:35,112 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb70c6fff1341daa8ebf27b30dc077f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731577768970 2024-11-14T09:49:35,112 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b05506e3b1446ababced466fd13f085, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731577771097 2024-11-14T09:49:35,112 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b9ad9eff434cd88f1b588799babae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731577771851 2024-11-14T09:49:35,114 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/A of 9e80b15335c1b44bab6a91f9a0dcea20 into 73df2b818a5e4216bf1e30a475cf2f76(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:35,114 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:35,115 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/A, priority=13, startTime=1731577774680; duration=0sec 2024-11-14T09:49:35,115 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:35,115 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:A 2024-11-14T09:49:35,119 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9e80b15335c1b44bab6a91f9a0dcea20#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:35,119 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/7a093083e3914a589741672e5ac1f7e9 is 50, key is test_row_0/C:col10/1731577771861/Put/seqid=0 2024-11-14T09:49:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742244_1420 (size=12949) 2024-11-14T09:49:35,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:35,134 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146ae4f2267b81436fa1d513c01de5cbb8_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ae4f2267b81436fa1d513c01de5cbb8_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:35,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/087241562f1d4c5fa23c213cb5a9e086, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:35,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/087241562f1d4c5fa23c213cb5a9e086 is 175, key is test_row_0/A:col10/1731577773020/Put/seqid=0 2024-11-14T09:49:35,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742245_1421 (size=31255) 2024-11-14T09:49:35,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:35,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. as already flushing 2024-11-14T09:49:35,240 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-14T09:49:35,240 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-14T09:49:35,240 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:35,240 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:35,240 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:35,240 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:35,240 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:35,241 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:35,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:35,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:35,526 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/7a093083e3914a589741672e5ac1f7e9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/7a093083e3914a589741672e5ac1f7e9 2024-11-14T09:49:35,530 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9e80b15335c1b44bab6a91f9a0dcea20/C of 9e80b15335c1b44bab6a91f9a0dcea20 into 7a093083e3914a589741672e5ac1f7e9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:35,530 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:35,530 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., storeName=9e80b15335c1b44bab6a91f9a0dcea20/C, priority=13, startTime=1731577774680; duration=0sec 2024-11-14T09:49:35,530 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:35,530 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9e80b15335c1b44bab6a91f9a0dcea20:C 2024-11-14T09:49:35,539 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/087241562f1d4c5fa23c213cb5a9e086 2024-11-14T09:49:35,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/cd22273ff9a7452eaf5165c68066d4cc is 50, key is test_row_0/B:col10/1731577773020/Put/seqid=0 2024-11-14T09:49:35,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742246_1422 (size=12301) 2024-11-14T09:49:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:35,948 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/cd22273ff9a7452eaf5165c68066d4cc 2024-11-14T09:49:35,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 is 50, key is test_row_0/C:col10/1731577773020/Put/seqid=0 2024-11-14T09:49:35,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742247_1423 (size=12301) 2024-11-14T09:49:36,356 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 2024-11-14T09:49:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/087241562f1d4c5fa23c213cb5a9e086 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/087241562f1d4c5fa23c213cb5a9e086 2024-11-14T09:49:36,364 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/087241562f1d4c5fa23c213cb5a9e086, entries=150, sequenceid=290, filesize=30.5 K 2024-11-14T09:49:36,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/cd22273ff9a7452eaf5165c68066d4cc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/cd22273ff9a7452eaf5165c68066d4cc 2024-11-14T09:49:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/cd22273ff9a7452eaf5165c68066d4cc, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T09:49:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 2024-11-14T09:49:36,374 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/2dedeeab5f8d43199bbc3f1f6b4d9a58, entries=150, sequenceid=290, filesize=12.0 K 2024-11-14T09:49:36,375 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1655ms, sequenceid=290, compaction requested=false 2024-11-14T09:49:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 9e80b15335c1b44bab6a91f9a0dcea20: 2024-11-14T09:49:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:36,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T09:49:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T09:49:36,376 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:49:36,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T09:49:36,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7280 sec 2024-11-14T09:49:36,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7330 sec 2024-11-14T09:49:37,196 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-14T09:49:37,196 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:37,196 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:37,196 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:37,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T09:49:37,805 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:39,476 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-14T09:49:39,476 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:39,476 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,476 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:39,517 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-14T09:49:39,517 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:49:39,517 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,517 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1550 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4641 rows 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1555 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4657 rows 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1560 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4679 rows 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1551 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4645 rows 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1551 2024-11-14T09:49:39,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4649 rows 2024-11-14T09:49:39,518 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:49:39,518 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:49:39,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:39,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:49:39,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:49:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:39,522 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577779522"}]},"ts":"1731577779522"} 2024-11-14T09:49:39,523 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:49:39,523 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:49:39,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:49:39,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, UNASSIGN}] 2024-11-14T09:49:39,525 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, UNASSIGN 2024-11-14T09:49:39,526 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:39,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, UNASSIGN because future has completed 2024-11-14T09:49:39,527 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:49:39,527 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:39,679 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:39,679 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:49:39,680 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 9e80b15335c1b44bab6a91f9a0dcea20, disabling compactions & flushes 2024-11-14T09:49:39,680 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:39,680 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:39,680 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. after waiting 0 ms 2024-11-14T09:49:39,680 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:39,680 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 9e80b15335c1b44bab6a91f9a0dcea20 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T09:49:39,685 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111406ff5c5aeceb4a14a5ddbc48930adbc4_9e80b15335c1b44bab6a91f9a0dcea20 is 50, key is test_row_0/A:col10/1731577779515/Put/seqid=0 2024-11-14T09:49:39,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742248_1424 (size=12454) 2024-11-14T09:49:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:40,090 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:40,096 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111406ff5c5aeceb4a14a5ddbc48930adbc4_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406ff5c5aeceb4a14a5ddbc48930adbc4_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:40,098 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/601cfcb9207b484a8ffcddfa698728a2, store: [table=TestAcidGuarantees family=A region=9e80b15335c1b44bab6a91f9a0dcea20] 2024-11-14T09:49:40,098 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/601cfcb9207b484a8ffcddfa698728a2 is 175, key is test_row_0/A:col10/1731577779515/Put/seqid=0 2024-11-14T09:49:40,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742249_1425 (size=31255) 2024-11-14T09:49:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:40,504 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/601cfcb9207b484a8ffcddfa698728a2 2024-11-14T09:49:40,509 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bad8f2208ce8445ead1fcdea1aa7fe11 is 50, key is test_row_0/B:col10/1731577779515/Put/seqid=0 2024-11-14T09:49:40,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742250_1426 (size=12301) 2024-11-14T09:49:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:40,913 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bad8f2208ce8445ead1fcdea1aa7fe11 2024-11-14T09:49:40,919 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ccaf7e7bea3a43a79ae29faab60e9610 is 50, key is test_row_0/C:col10/1731577779515/Put/seqid=0 2024-11-14T09:49:40,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742251_1427 (size=12301) 2024-11-14T09:49:41,084 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26] to archive 2024-11-14T09:49:41,085 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:41,086 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/d7336a274b17420f89a0153e93715cf3 2024-11-14T09:49:41,087 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/35603f3f8fa341fe867c904221efd760 2024-11-14T09:49:41,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/39eac5ac99a649edaadb8b21b7391493 2024-11-14T09:49:41,089 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f140705947294ab081d5c67829c24497 2024-11-14T09:49:41,090 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/51ad6dc158094fca91898014f6814bf0 2024-11-14T09:49:41,091 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/4ad471b5b4964478acd4e1e9b810f891 2024-11-14T09:49:41,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/ce59542a7cc04205b2792c3a8f213a11 2024-11-14T09:49:41,092 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/0305929676834c30933499e97478a4e2 2024-11-14T09:49:41,093 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6f0a470941dd415da66ed062be241f80 2024-11-14T09:49:41,094 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f3ca17243b6d4a4fa225e80c211009c0 2024-11-14T09:49:41,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/3ddd8eb9b6d14ae4b16434f959d84fb9 2024-11-14T09:49:41,095 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8aedcf9180d042389ae46f8d86552c1a 2024-11-14T09:49:41,096 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e3f3078e65ca47e09a353e166cc62daa 2024-11-14T09:49:41,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/f5fb5cf645704c26906e161e26c4e74e 2024-11-14T09:49:41,097 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/e14d40170dc8461d90d290dd7c26ffd6 2024-11-14T09:49:41,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/dbd0923709da40cd8e1b224780553d49 2024-11-14T09:49:41,098 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/8bf96f05dd634ff2be7bedb56b65f5c4 2024-11-14T09:49:41,099 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/6364b3f216d44597b62e3b3b55bc9aaa 2024-11-14T09:49:41,100 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/bf1c48d2faa849f3af9397b5e262ea26 2024-11-14T09:49:41,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/15d0540e3b6e4571838913aa2b5c6592, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/eccc1c89abb747f8b8c748c5c5a764d6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/01d4e1eac27b44b18e889e2e321c50ae, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/dae5813865b44be2b01c9f0341b2797a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/404ae03e5c9e4ff3b5f9972eab3d4e6c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f] to archive 2024-11-14T09:49:41,102 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:41,103 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/69eefc8e4cd341c4930dd02e5aea32ff 2024-11-14T09:49:41,104 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3f18ed4e1e43414c8b4d8bb6f507637a 2024-11-14T09:49:41,105 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/15d0540e3b6e4571838913aa2b5c6592 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/15d0540e3b6e4571838913aa2b5c6592 2024-11-14T09:49:41,105 WARN [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-11-14T09:49:41,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6040dd93e1034216a563457a1ac4a0e1 2024-11-14T09:49:41,106 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/b6416178d829425d95355ca33f025620 2024-11-14T09:49:41,107 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/eccc1c89abb747f8b8c748c5c5a764d6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/eccc1c89abb747f8b8c748c5c5a764d6 2024-11-14T09:49:41,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bf2881495d2d4757a96edbebf1ba7b30 2024-11-14T09:49:41,108 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/10c2ff8dd1f94adfbd37350fe5e2fe7b 2024-11-14T09:49:41,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/01d4e1eac27b44b18e889e2e321c50ae to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/01d4e1eac27b44b18e889e2e321c50ae 2024-11-14T09:49:41,109 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/581f1855f7ec4909add136625a5a9911 2024-11-14T09:49:41,110 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/41f427c7025f493c8b540797e069bbf8 2024-11-14T09:49:41,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/990ed907338d46b38d37b802ae4fa0fd 2024-11-14T09:49:41,111 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/dae5813865b44be2b01c9f0341b2797a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/dae5813865b44be2b01c9f0341b2797a 2024-11-14T09:49:41,112 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/1571381210d6494f8b8f68b358a7645e 2024-11-14T09:49:41,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/c128f4971a094e1ca2fa0d8c85744f58 2024-11-14T09:49:41,113 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/404ae03e5c9e4ff3b5f9972eab3d4e6c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/404ae03e5c9e4ff3b5f9972eab3d4e6c 2024-11-14T09:49:41,114 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/3caad0d3e21a426fa9c02d5daeeb6363 2024-11-14T09:49:41,115 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/058753070f0f42bc8bbf849cf48c50e0 2024-11-14T09:49:41,116 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/0ba27f8912b9406b85b1b065ad8c147f 2024-11-14T09:49:41,118 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/da9c4747d428481e88eedd3819ae7cb7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5b962ef89c654f158c1957e72f4ad7f3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/6b268403a8c54b72b88d4faecdbfe7c1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5bd7f9843e814143b690ce34bde9754a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/cdb70c6fff1341daa8ebf27b30dc077f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7] to archive 2024-11-14T09:49:41,119 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:49:41,120 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/63aab05b05e74f7fbd4847f25c5cbcef 2024-11-14T09:49:41,121 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/f525b0eaf3b14870aea9fea7792fb7f4 2024-11-14T09:49:41,122 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/da9c4747d428481e88eedd3819ae7cb7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/da9c4747d428481e88eedd3819ae7cb7 2024-11-14T09:49:41,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/0c73b3069c524d89be6dc669d9d6bc3b 2024-11-14T09:49:41,123 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/195f6ab9b080424696e7db175070bbcd 2024-11-14T09:49:41,124 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5b962ef89c654f158c1957e72f4ad7f3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5b962ef89c654f158c1957e72f4ad7f3 2024-11-14T09:49:41,125 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/a589a39cb0794e88b293aa5ca9bf8d62 2024-11-14T09:49:41,126 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/797bb7eaa24a430f8fb9c7a53e21a1cd 2024-11-14T09:49:41,127 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/6b268403a8c54b72b88d4faecdbfe7c1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/6b268403a8c54b72b88d4faecdbfe7c1 2024-11-14T09:49:41,128 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/77f29657f93e4349b20233b4e7e9c502 2024-11-14T09:49:41,129 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/40f060cd704c401ab378a2df36d2ead8 2024-11-14T09:49:41,130 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4bb7aff0bb7c4cc2b51231b380a7a4cc 2024-11-14T09:49:41,131 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5bd7f9843e814143b690ce34bde9754a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/5bd7f9843e814143b690ce34bde9754a 2024-11-14T09:49:41,132 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/c013c17893074cb486d859790ef4165d 2024-11-14T09:49:41,133 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/4c0633dc32bc40e6a9c57fdebf68d646 2024-11-14T09:49:41,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/cdb70c6fff1341daa8ebf27b30dc077f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/cdb70c6fff1341daa8ebf27b30dc077f 2024-11-14T09:49:41,134 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ab88425667444684a66de1b0a1e6c585 2024-11-14T09:49:41,135 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/1b05506e3b1446ababced466fd13f085 2024-11-14T09:49:41,136 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/defc576eb6b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/93b9ad9eff434cd88f1b588799babae7 2024-11-14T09:49:41,322 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ccaf7e7bea3a43a79ae29faab60e9610 2024-11-14T09:49:41,326 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/A/601cfcb9207b484a8ffcddfa698728a2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/601cfcb9207b484a8ffcddfa698728a2 2024-11-14T09:49:41,329 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/601cfcb9207b484a8ffcddfa698728a2, entries=150, sequenceid=301, filesize=30.5 K 2024-11-14T09:49:41,330 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/B/bad8f2208ce8445ead1fcdea1aa7fe11 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bad8f2208ce8445ead1fcdea1aa7fe11 2024-11-14T09:49:41,333 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bad8f2208ce8445ead1fcdea1aa7fe11, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T09:49:41,334 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/.tmp/C/ccaf7e7bea3a43a79ae29faab60e9610 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ccaf7e7bea3a43a79ae29faab60e9610 2024-11-14T09:49:41,338 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ccaf7e7bea3a43a79ae29faab60e9610, entries=150, sequenceid=301, filesize=12.0 K 2024-11-14T09:49:41,339 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1659ms, sequenceid=301, compaction requested=true 2024-11-14T09:49:41,352 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits/304.seqid, newMaxSeqId=304, maxSeqId=4 2024-11-14T09:49:41,353 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. 2024-11-14T09:49:41,353 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 9e80b15335c1b44bab6a91f9a0dcea20: Waiting for close lock at 1731577779679Running coprocessor pre-close hooks at 1731577779679Disabling compacts and flushes for region at 1731577779679Disabling writes for close at 1731577779680 (+1 ms)Obtaining lock to block concurrent updates at 1731577779680Preparing flush snapshotting stores in 9e80b15335c1b44bab6a91f9a0dcea20 at 1731577779680Finished memstore snapshotting TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731577779680Flushing stores of TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20. at 1731577779680Flushing 9e80b15335c1b44bab6a91f9a0dcea20/A: creating writer at 1731577779680Flushing mob file 9e80b15335c1b44bab6a91f9a0dcea20/A: appending metadata at 1731577779685 (+5 ms)Flushing mob file 9e80b15335c1b44bab6a91f9a0dcea20/A: closing flushed file at 1731577779685Flushing 9e80b15335c1b44bab6a91f9a0dcea20/A: appending metadata at 1731577780098 (+413 ms)Flushing 9e80b15335c1b44bab6a91f9a0dcea20/A: closing flushed file at 1731577780098Flushing 9e80b15335c1b44bab6a91f9a0dcea20/B: creating writer at 1731577780507 (+409 ms)Flushing 9e80b15335c1b44bab6a91f9a0dcea20/B: appending metadata at 1731577780509 (+2 ms)Flushing 9e80b15335c1b44bab6a91f9a0dcea20/B: closing flushed file at 1731577780509Flushing 9e80b15335c1b44bab6a91f9a0dcea20/C: creating writer at 1731577780916 (+407 ms)Flushing 9e80b15335c1b44bab6a91f9a0dcea20/C: appending metadata at 1731577780918 (+2 ms)Flushing 9e80b15335c1b44bab6a91f9a0dcea20/C: closing flushed file at 1731577780918Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@171fb5ee: reopening flushed file at 1731577781325 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bb25ddb: reopening flushed file at 1731577781329 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c6e8e2c: reopening flushed file at 1731577781334 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9e80b15335c1b44bab6a91f9a0dcea20 in 1659ms, sequenceid=301, compaction requested=true at 1731577781339 (+5 ms)Writing region close event to WAL at 1731577781348 (+9 ms)Running coprocessor post-close hooks at 1731577781353 (+5 ms)Closed at 1731577781353 2024-11-14T09:49:41,355 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,356 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=9e80b15335c1b44bab6a91f9a0dcea20, regionState=CLOSED 2024-11-14T09:49:41,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:41,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T09:49:41,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e80b15335c1b44bab6a91f9a0dcea20, server=defc576eb6b7,40243,1731577658084 in 1.8310 sec 2024-11-14T09:49:41,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-14T09:49:41,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9e80b15335c1b44bab6a91f9a0dcea20, UNASSIGN in 1.8350 sec 2024-11-14T09:49:41,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T09:49:41,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8380 sec 2024-11-14T09:49:41,365 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577781365"}]},"ts":"1731577781365"} 2024-11-14T09:49:41,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:49:41,366 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:49:41,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8490 sec 2024-11-14T09:49:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T09:49:41,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:41,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:49:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,668 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,669 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:49:41,670 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,672 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits] 2024-11-14T09:49:41,674 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/087241562f1d4c5fa23c213cb5a9e086 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/087241562f1d4c5fa23c213cb5a9e086 2024-11-14T09:49:41,676 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/601cfcb9207b484a8ffcddfa698728a2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/601cfcb9207b484a8ffcddfa698728a2 2024-11-14T09:49:41,677 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/73df2b818a5e4216bf1e30a475cf2f76 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/A/73df2b818a5e4216bf1e30a475cf2f76 2024-11-14T09:49:41,679 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6e77fb8986ef430e8d2c9e7631355412 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/6e77fb8986ef430e8d2c9e7631355412 2024-11-14T09:49:41,680 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bad8f2208ce8445ead1fcdea1aa7fe11 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/bad8f2208ce8445ead1fcdea1aa7fe11 2024-11-14T09:49:41,681 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/cd22273ff9a7452eaf5165c68066d4cc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/B/cd22273ff9a7452eaf5165c68066d4cc 2024-11-14T09:49:41,683 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/2dedeeab5f8d43199bbc3f1f6b4d9a58 2024-11-14T09:49:41,684 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/7a093083e3914a589741672e5ac1f7e9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/7a093083e3914a589741672e5ac1f7e9 2024-11-14T09:49:41,685 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ccaf7e7bea3a43a79ae29faab60e9610 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/C/ccaf7e7bea3a43a79ae29faab60e9610 2024-11-14T09:49:41,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits/304.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20/recovered.edits/304.seqid 2024-11-14T09:49:41,688 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,688 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:49:41,689 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:49:41,689 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:49:41,692 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406ff5c5aeceb4a14a5ddbc48930adbc4_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111406ff5c5aeceb4a14a5ddbc48930adbc4_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,694 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111427ad65c5fc3a4c0fb64ef5a1025952d3_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111427ad65c5fc3a4c0fb64ef5a1025952d3_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,695 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acd08abb1b94f128843db1101bc9c4a_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142acd08abb1b94f128843db1101bc9c4a_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,696 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114389488f44e024b839122111092cd01d5_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114389488f44e024b839122111092cd01d5_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,697 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143acbfa77740f4c2a8205fb60ca4941e1_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143acbfa77740f4c2a8205fb60ca4941e1_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,698 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442abd86a8df14bb7bca5c8f508e91140_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111442abd86a8df14bb7bca5c8f508e91140_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,699 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144524fcc5ca6447989736d7fd5c1a338e_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144524fcc5ca6447989736d7fd5c1a338e_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,700 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114597fdab411b14922bbbef9c8f78d939d_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114597fdab411b14922bbbef9c8f78d939d_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,701 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465e65a2ab06246389cc66d5ea24d6a2d_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465e65a2ab06246389cc66d5ea24d6a2d_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,703 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ae4f2267b81436fa1d513c01de5cbb8_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146ae4f2267b81436fa1d513c01de5cbb8_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,704 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149239152f32d644fbbe4b70c0b22cccef_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149239152f32d644fbbe4b70c0b22cccef_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,705 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149e69f19fd7934ad29fc231f66267d4f1_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149e69f19fd7934ad29fc231f66267d4f1_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3d7cea7d35f477c9fd6c9b01685fc37_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d3d7cea7d35f477c9fd6c9b01685fc37_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,706 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e437a35cf3804fac90fc86265a10bff0_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e437a35cf3804fac90fc86265a10bff0_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,707 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f14443a4d8464dada34f3946edad3bb5_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f14443a4d8464dada34f3946edad3bb5_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,708 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa9f03fa367b4fa2b44f1c440c53307a_9e80b15335c1b44bab6a91f9a0dcea20 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fa9f03fa367b4fa2b44f1c440c53307a_9e80b15335c1b44bab6a91f9a0dcea20 2024-11-14T09:49:41,708 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:49:41,710 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,712 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:49:41,714 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:49:41,716 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,716 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:49:41,716 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577781716"}]},"ts":"9223372036854775807"} 2024-11-14T09:49:41,719 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:49:41,719 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9e80b15335c1b44bab6a91f9a0dcea20, NAME => 'TestAcidGuarantees,,1731577751711.9e80b15335c1b44bab6a91f9a0dcea20.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:49:41,719 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:49:41,719 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577781719"}]},"ts":"9223372036854775807"} 2024-11-14T09:49:41,722 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:49:41,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,724 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-14T09:49:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-14T09:49:41,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:49:41,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:41,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=492 (was 469) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4476 (was 4188) - AvailableMemoryMB LEAK? - 2024-11-14T09:49:41,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=492, ProcessCount=11, AvailableMemoryMB=4476 2024-11-14T09:49:41,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:49:41,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:49:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:49:41,797 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:49:41,797 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:41,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-14T09:49:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:41,798 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:49:41,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742252_1428 (size=959) 2024-11-14T09:49:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:42,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:42,205 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:49:42,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742253_1429 (size=53) 2024-11-14T09:49:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:42,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:49:42,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76f327b9ee0a90edb9e6efb3d43a25dd, disabling compactions & flushes 2024-11-14T09:49:42,621 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. after waiting 0 ms 2024-11-14T09:49:42,621 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,622 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,622 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76f327b9ee0a90edb9e6efb3d43a25dd: Waiting for close lock at 1731577782621Disabling compacts and flushes for region at 1731577782621Disabling writes for close at 1731577782621Writing region close event to WAL at 1731577782621Closed at 1731577782621 2024-11-14T09:49:42,623 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:49:42,623 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577782623"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577782623"}]},"ts":"1731577782623"} 2024-11-14T09:49:42,629 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:49:42,629 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:49:42,630 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577782629"}]},"ts":"1731577782629"} 2024-11-14T09:49:42,631 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:49:42,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, ASSIGN}] 2024-11-14T09:49:42,632 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, ASSIGN 2024-11-14T09:49:42,632 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:49:42,783 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=76f327b9ee0a90edb9e6efb3d43a25dd, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:42,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, ASSIGN because future has completed 2024-11-14T09:49:42,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:49:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:42,941 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,941 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:49:42,942 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,942 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:49:42,942 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,942 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,943 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,945 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76f327b9ee0a90edb9e6efb3d43a25dd columnFamilyName A 2024-11-14T09:49:42,945 DEBUG [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:42,945 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(327): Store=76f327b9ee0a90edb9e6efb3d43a25dd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:42,946 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,947 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76f327b9ee0a90edb9e6efb3d43a25dd columnFamilyName B 2024-11-14T09:49:42,947 DEBUG [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:42,947 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(327): Store=76f327b9ee0a90edb9e6efb3d43a25dd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:42,947 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,948 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76f327b9ee0a90edb9e6efb3d43a25dd columnFamilyName C 2024-11-14T09:49:42,949 DEBUG [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:49:42,949 INFO [StoreOpener-76f327b9ee0a90edb9e6efb3d43a25dd-1 {}] regionserver.HStore(327): Store=76f327b9ee0a90edb9e6efb3d43a25dd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:49:42,949 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,950 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,950 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,951 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,951 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,952 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:49:42,953 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,955 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:49:42,955 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 76f327b9ee0a90edb9e6efb3d43a25dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59052344, jitterRate=-0.12005150318145752}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:49:42,955 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:42,956 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 76f327b9ee0a90edb9e6efb3d43a25dd: Running coprocessor pre-open hook at 1731577782942Writing region info on filesystem at 1731577782942Initializing all the Stores at 1731577782943 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577782943Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577782943Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577782943Cleaning up temporary data from old regions at 1731577782951 (+8 ms)Running coprocessor post-open hooks at 1731577782955 (+4 ms)Region opened successfully at 1731577782956 (+1 ms) 2024-11-14T09:49:42,957 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., pid=112, masterSystemTime=1731577782938 2024-11-14T09:49:42,959 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,959 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:42,960 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=76f327b9ee0a90edb9e6efb3d43a25dd, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:42,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:49:42,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-14T09:49:42,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 in 177 msec 2024-11-14T09:49:42,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-14T09:49:42,965 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, ASSIGN in 333 msec 2024-11-14T09:49:42,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:49:42,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577782966"}]},"ts":"1731577782966"} 2024-11-14T09:49:42,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:49:42,969 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:49:42,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-14T09:49:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-14T09:49:43,945 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:43,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed34e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7be34e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,949 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:43,951 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:43,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20c8932c 2024-11-14T09:49:43,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:49:43,953 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:49:43,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,955 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c05178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,956 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8b6d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,958 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f11ad31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,960 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68137b88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,965 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e812d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,966 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255ca9ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,968 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423dfe08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51a33452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,972 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e35dac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0e3971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130dbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390af7bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,979 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2408ea26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,981 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37d3c6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,985 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdabe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ca163b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,993 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb50fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,995 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cb967e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:49:43,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:49:43,997 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7a932c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:49:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:49:43,999 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:49:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8ec433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:49:44,000 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,000 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,001 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,002 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,003 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,004 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,006 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,007 DEBUG [Thread-1949 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,008 DEBUG [Thread-1949 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,008 DEBUG [Thread-1949 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:44,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-14T09:49:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:44,011 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,012 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,013 DEBUG [Thread-1945 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,014 DEBUG [Thread-1945 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,014 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:44,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:44,015 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:49:44,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:49:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:49:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:49:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2] 2024-11-14T09:49:44,023 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:44,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T09:49:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:44,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/065576739fc64c65a4712610b45ebc9e is 50, key is test_row_0/A:col10/1731577784021/Put/seqid=0 2024-11-14T09:49:44,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742254_1430 (size=12001) 2024-11-14T09:49:44,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/065576739fc64c65a4712610b45ebc9e 2024-11-14T09:49:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577844069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577844070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577844071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577844074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577844073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/3d971fcfce5d4bddb4cec1230349af69 is 50, key is test_row_0/B:col10/1731577784021/Put/seqid=0 2024-11-14T09:49:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:44,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742255_1431 (size=12001) 2024-11-14T09:49:44,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T09:49:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577844185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577844185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577844185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577844194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577844195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:44,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T09:49:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577844405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577844405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577844405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577844405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577844405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T09:49:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:44,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/3d971fcfce5d4bddb4cec1230349af69 2024-11-14T09:49:44,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/eaeedf25c1364015b208637c4c38b1b6 is 50, key is test_row_0/C:col10/1731577784021/Put/seqid=0 2024-11-14T09:49:44,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742256_1432 (size=12001) 2024-11-14T09:49:44,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/eaeedf25c1364015b208637c4c38b1b6 2024-11-14T09:49:44,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/065576739fc64c65a4712610b45ebc9e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e 2024-11-14T09:49:44,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:49:44,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/3d971fcfce5d4bddb4cec1230349af69 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69 2024-11-14T09:49:44,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:49:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/eaeedf25c1364015b208637c4c38b1b6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6 2024-11-14T09:49:44,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T09:49:44,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 76f327b9ee0a90edb9e6efb3d43a25dd in 568ms, sequenceid=13, compaction requested=false 2024-11-14T09:49:44,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:44,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-14T09:49:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:44,635 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:49:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f36585fe43474109b644f9f56029d0bd is 50, key is test_row_0/A:col10/1731577784072/Put/seqid=0 2024-11-14T09:49:44,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742257_1433 (size=12001) 2024-11-14T09:49:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:44,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577844724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577844724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577844731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577844732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577844733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577844834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577844835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577844845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577844846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577844846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:44,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577845045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577845054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577845054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577845054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,071 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f36585fe43474109b644f9f56029d0bd 2024-11-14T09:49:45,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/f7acb146ff4d4f228a16a896e6faf2bc is 50, key is test_row_0/B:col10/1731577784072/Put/seqid=0 2024-11-14T09:49:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742258_1434 (size=12001) 2024-11-14T09:49:45,083 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/f7acb146ff4d4f228a16a896e6faf2bc 2024-11-14T09:49:45,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/7183e5a8f23344f0a413f8b3048fc91b is 50, key is test_row_0/C:col10/1731577784072/Put/seqid=0 2024-11-14T09:49:45,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742259_1435 (size=12001) 2024-11-14T09:49:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577845355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577845355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577845364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577845364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577845365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,502 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/7183e5a8f23344f0a413f8b3048fc91b 2024-11-14T09:49:45,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f36585fe43474109b644f9f56029d0bd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd 2024-11-14T09:49:45,514 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:49:45,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/f7acb146ff4d4f228a16a896e6faf2bc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc 2024-11-14T09:49:45,518 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:49:45,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/7183e5a8f23344f0a413f8b3048fc91b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b 2024-11-14T09:49:45,523 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T09:49:45,524 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 76f327b9ee0a90edb9e6efb3d43a25dd in 889ms, sequenceid=37, compaction requested=false 2024-11-14T09:49:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-14T09:49:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-14T09:49:45,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T09:49:45,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5110 sec 2024-11-14T09:49:45,528 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5170 sec 2024-11-14T09:49:45,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:45,868 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:45,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:49:45,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8c49421285e848ba8b6acbecabbe2c55 is 50, key is test_row_0/A:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:45,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742260_1436 (size=11997) 2024-11-14T09:49:45,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577845902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577845903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577845908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577845910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:45,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577845911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:45,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577846015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577846015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577846015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577846024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577846024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-14T09:49:46,155 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:46,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T09:49:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:46,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:46,157 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:46,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:46,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577846224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577846224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577846224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577846235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577846235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:46,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8c49421285e848ba8b6acbecabbe2c55 2024-11-14T09:49:46,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/be9ab4bf703648c9896fc845f9c27abf is 50, key is test_row_0/B:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:46,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742261_1437 (size=9657) 2024-11-14T09:49:46,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:49:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:49:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577846534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577846544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577846544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577846555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577846555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:46,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:49:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:46,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:46,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/be9ab4bf703648c9896fc845f9c27abf 2024-11-14T09:49:46,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/45b558b731ee4f0b9c6c286f444795f1 is 50, key is test_row_0/C:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:46,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742262_1438 (size=9657) 2024-11-14T09:49:46,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/45b558b731ee4f0b9c6c286f444795f1 2024-11-14T09:49:46,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8c49421285e848ba8b6acbecabbe2c55 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55 2024-11-14T09:49:46,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T09:49:46,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/be9ab4bf703648c9896fc845f9c27abf as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf 2024-11-14T09:49:46,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf, entries=100, sequenceid=50, filesize=9.4 K 2024-11-14T09:49:46,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/45b558b731ee4f0b9c6c286f444795f1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1 2024-11-14T09:49:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1, entries=100, sequenceid=50, filesize=9.4 K 2024-11-14T09:49:46,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 76f327b9ee0a90edb9e6efb3d43a25dd in 842ms, sequenceid=50, compaction requested=true 2024-11-14T09:49:46,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:46,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:46,711 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:46,711 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,711 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,712 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=35.2 K 2024-11-14T09:49:46,712 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=32.9 K 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 065576739fc64c65a4712610b45ebc9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577784009 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d971fcfce5d4bddb4cec1230349af69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577784009 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f36585fe43474109b644f9f56029d0bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577784058 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f7acb146ff4d4f228a16a896e6faf2bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577784058 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c49421285e848ba8b6acbecabbe2c55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784730 2024-11-14T09:49:46,712 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting be9ab4bf703648c9896fc845f9c27abf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784731 2024-11-14T09:49:46,720 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:46,720 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/3c3f4fa886514a20a29db435deb82374 is 50, key is test_row_0/B:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:46,722 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:46,722 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/250e8586f86e41aeb2578b0dde9387e7 is 50, key is test_row_0/A:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:46,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742263_1439 (size=12104) 2024-11-14T09:49:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742264_1440 (size=12104) 2024-11-14T09:49:46,738 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/3c3f4fa886514a20a29db435deb82374 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3c3f4fa886514a20a29db435deb82374 2024-11-14T09:49:46,743 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 3c3f4fa886514a20a29db435deb82374(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:46,743 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577786711; duration=0sec 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:46,743 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:46,743 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,744 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=32.9 K 2024-11-14T09:49:46,745 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting eaeedf25c1364015b208637c4c38b1b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731577784009 2024-11-14T09:49:46,745 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7183e5a8f23344f0a413f8b3048fc91b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731577784058 2024-11-14T09:49:46,745 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b558b731ee4f0b9c6c286f444795f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784731 2024-11-14T09:49:46,753 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#371 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:46,753 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/260916778ece4c6e915a846c7ad42906 is 50, key is test_row_0/C:col10/1731577785867/Put/seqid=0 2024-11-14T09:49:46,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742265_1441 (size=12104) 2024-11-14T09:49:46,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T09:49:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:46,768 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:49:46,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/790d2eb978964ff1b3e2191f15543a9f is 50, key is test_row_0/A:col10/1731577785901/Put/seqid=0 2024-11-14T09:49:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:46,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742266_1442 (size=12001) 2024-11-14T09:49:46,785 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/790d2eb978964ff1b3e2191f15543a9f 2024-11-14T09:49:46,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/54dfd340c1ee4737838906775379a429 is 50, key is test_row_0/B:col10/1731577785901/Put/seqid=0 2024-11-14T09:49:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742267_1443 (size=12001) 2024-11-14T09:49:46,879 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:49:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:47,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577847075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577847075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577847083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577847083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577847084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,138 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/250e8586f86e41aeb2578b0dde9387e7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/250e8586f86e41aeb2578b0dde9387e7 2024-11-14T09:49:47,142 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 250e8586f86e41aeb2578b0dde9387e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:47,142 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:47,142 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577786710; duration=0sec 2024-11-14T09:49:47,142 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:47,142 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:47,164 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/260916778ece4c6e915a846c7ad42906 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/260916778ece4c6e915a846c7ad42906 2024-11-14T09:49:47,167 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 260916778ece4c6e915a846c7ad42906(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:47,167 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:47,167 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577786711; duration=0sec 2024-11-14T09:49:47,168 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:47,168 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:47,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577847194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577847194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577847194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,206 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/54dfd340c1ee4737838906775379a429 2024-11-14T09:49:47,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ce48a6865d494e4895c2e28eef3ea782 is 50, key is test_row_0/C:col10/1731577785901/Put/seqid=0 2024-11-14T09:49:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742268_1444 (size=12001) 2024-11-14T09:49:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:47,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577847404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577847404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577847404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,615 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ce48a6865d494e4895c2e28eef3ea782 2024-11-14T09:49:47,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/790d2eb978964ff1b3e2191f15543a9f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f 2024-11-14T09:49:47,623 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T09:49:47,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/54dfd340c1ee4737838906775379a429 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429 2024-11-14T09:49:47,628 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T09:49:47,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ce48a6865d494e4895c2e28eef3ea782 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782 2024-11-14T09:49:47,632 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T09:49:47,633 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 76f327b9ee0a90edb9e6efb3d43a25dd in 865ms, sequenceid=75, compaction requested=false 2024-11-14T09:49:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T09:49:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T09:49:47,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T09:49:47,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4770 sec 2024-11-14T09:49:47,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4810 sec 2024-11-14T09:49:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:47,718 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:47,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:47,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8a2c3f2af0284d44b10e1dc8ec75c185 is 50, key is test_row_0/A:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:47,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742269_1445 (size=14341) 2024-11-14T09:49:47,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577847759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577847760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577847761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577847875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577847875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577847875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:47,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577848094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577848095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,102 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577848095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577848095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577848095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8a2c3f2af0284d44b10e1dc8ec75c185 2024-11-14T09:49:48,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/4d678c956d5b44d9a57ce696fa907c53 is 50, key is test_row_0/B:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:48,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742270_1446 (size=12001) 2024-11-14T09:49:48,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/4d678c956d5b44d9a57ce696fa907c53 2024-11-14T09:49:48,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/0407591a827746f0954a3b39d3a6b182 is 50, key is test_row_0/C:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:48,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742271_1447 (size=12001) 2024-11-14T09:49:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T09:49:48,305 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:48,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T09:49:48,307 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:49:48,308 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:48,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:48,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577848404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:49:48,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577848415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577848415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:49:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:48,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/0407591a827746f0954a3b39d3a6b182 2024-11-14T09:49:48,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/8a2c3f2af0284d44b10e1dc8ec75c185 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185 2024-11-14T09:49:48,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185, entries=200, sequenceid=92, filesize=14.0 K 2024-11-14T09:49:48,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/4d678c956d5b44d9a57ce696fa907c53 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53 2024-11-14T09:49:48,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T09:49:48,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/0407591a827746f0954a3b39d3a6b182 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182 2024-11-14T09:49:48,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T09:49:48,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 76f327b9ee0a90edb9e6efb3d43a25dd in 848ms, sequenceid=92, compaction requested=true 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:48,566 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:48,566 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:48,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:48,567 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:48,567 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:48,567 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,567 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3c3f4fa886514a20a29db435deb82374, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=35.3 K 2024-11-14T09:49:48,567 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:48,567 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:48,567 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,567 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/250e8586f86e41aeb2578b0dde9387e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=37.5 K 2024-11-14T09:49:48,567 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3f4fa886514a20a29db435deb82374, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784058 2024-11-14T09:49:48,568 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250e8586f86e41aeb2578b0dde9387e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784058 2024-11-14T09:49:48,568 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 54dfd340c1ee4737838906775379a429, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577785901 2024-11-14T09:49:48,568 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d678c956d5b44d9a57ce696fa907c53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:48,568 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790d2eb978964ff1b3e2191f15543a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577785901 2024-11-14T09:49:48,569 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2c3f2af0284d44b10e1dc8ec75c185, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:48,582 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:48,582 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/061095f6a45343c99f344a39e72e7a5b is 50, key is test_row_0/A:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:48,582 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:48,583 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/50a149ef786d4d5cb11a4b42bb77d17e is 50, key is test_row_0/B:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:48,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742272_1448 (size=12207) 2024-11-14T09:49:48,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742273_1449 (size=12207) 2024-11-14T09:49:48,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T09:49:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,613 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:49:48,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/c51a32aa48a54f00b50e9c82221be0d7 is 50, key is test_row_0/A:col10/1731577787760/Put/seqid=0 2024-11-14T09:49:48,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742274_1450 (size=12001) 2024-11-14T09:49:48,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:49:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:48,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:49:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577848953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577848954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577848964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:48,991 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/061095f6a45343c99f344a39e72e7a5b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/061095f6a45343c99f344a39e72e7a5b 2024-11-14T09:49:48,995 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 061095f6a45343c99f344a39e72e7a5b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:48,995 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:48,995 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577788566; duration=0sec 2024-11-14T09:49:48,995 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:48,995 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:48,995 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:48,997 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:48,997 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:48,997 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:48,997 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/50a149ef786d4d5cb11a4b42bb77d17e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/50a149ef786d4d5cb11a4b42bb77d17e 2024-11-14T09:49:48,997 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/260916778ece4c6e915a846c7ad42906, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=35.3 K 2024-11-14T09:49:48,997 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 260916778ece4c6e915a846c7ad42906, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731577784058 2024-11-14T09:49:48,998 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce48a6865d494e4895c2e28eef3ea782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731577785901 2024-11-14T09:49:48,998 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0407591a827746f0954a3b39d3a6b182, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:49,001 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 50a149ef786d4d5cb11a4b42bb77d17e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:49,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,001 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577788566; duration=0sec 2024-11-14T09:49:49,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:49,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:49,004 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:49,004 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/86dbb1e290354c85ab4af032d2fc9aab is 50, key is test_row_0/C:col10/1731577787072/Put/seqid=0 2024-11-14T09:49:49,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742275_1451 (size=12207) 2024-11-14T09:49:49,020 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/c51a32aa48a54f00b50e9c82221be0d7 2024-11-14T09:49:49,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c3c87386a005403c83a94cfbf250ca7c is 50, key is test_row_0/B:col10/1731577787760/Put/seqid=0 2024-11-14T09:49:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742276_1452 (size=12001) 2024-11-14T09:49:49,032 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c3c87386a005403c83a94cfbf250ca7c 2024-11-14T09:49:49,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/a6f9b0306ba84cd0b2ce375f7acf8155 is 50, key is test_row_0/C:col10/1731577787760/Put/seqid=0 2024-11-14T09:49:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742277_1453 (size=12001) 2024-11-14T09:49:49,043 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/a6f9b0306ba84cd0b2ce375f7acf8155 2024-11-14T09:49:49,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/c51a32aa48a54f00b50e9c82221be0d7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7 2024-11-14T09:49:49,053 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T09:49:49,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c3c87386a005403c83a94cfbf250ca7c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c 2024-11-14T09:49:49,065 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T09:49:49,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/a6f9b0306ba84cd0b2ce375f7acf8155 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155 2024-11-14T09:49:49,071 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T09:49:49,073 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 76f327b9ee0a90edb9e6efb3d43a25dd in 460ms, sequenceid=113, compaction requested=false 2024-11-14T09:49:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T09:49:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T09:49:49,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T09:49:49,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 767 msec 2024-11-14T09:49:49,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 773 msec 2024-11-14T09:49:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:49,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:49,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:49:49,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/239b037afda74e86a3cc3b834635d206 is 50, key is test_row_0/A:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742278_1454 (size=14391) 2024-11-14T09:49:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577849137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577849138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577849145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577849255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577849255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577849265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/86dbb1e290354c85ab4af032d2fc9aab as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/86dbb1e290354c85ab4af032d2fc9aab 2024-11-14T09:49:49,426 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 86dbb1e290354c85ab4af032d2fc9aab(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:49,426 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,426 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577788566; duration=0sec 2024-11-14T09:49:49,426 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:49,426 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T09:49:49,445 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:49,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T09:49:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:49:49,447 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:49,448 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:49,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577849474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/239b037afda74e86a3cc3b834635d206 2024-11-14T09:49:49,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/6ad4ee91716a4faabb8fb319446e4ca5 is 50, key is test_row_0/B:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742279_1455 (size=12051) 2024-11-14T09:49:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:49:49,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T09:49:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:49,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T09:49:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:49:49,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577849785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577849794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:49,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:49,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/6ad4ee91716a4faabb8fb319446e4ca5 2024-11-14T09:49:49,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T09:49:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:49,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b9ab2d700fb5436483e44a91bb154b49 is 50, key is test_row_0/C:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742280_1456 (size=12051) 2024-11-14T09:49:49,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b9ab2d700fb5436483e44a91bb154b49 2024-11-14T09:49:49,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/239b037afda74e86a3cc3b834635d206 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206 2024-11-14T09:49:49,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206, entries=200, sequenceid=131, filesize=14.1 K 2024-11-14T09:49:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/6ad4ee91716a4faabb8fb319446e4ca5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5 2024-11-14T09:49:49,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T09:49:49,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b9ab2d700fb5436483e44a91bb154b49 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49 2024-11-14T09:49:49,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T09:49:49,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 76f327b9ee0a90edb9e6efb3d43a25dd in 851ms, sequenceid=131, compaction requested=true 2024-11-14T09:49:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,933 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:49,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:49,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:49,934 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:49,934 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:49,934 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:49,934 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,934 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/061095f6a45343c99f344a39e72e7a5b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=37.7 K 2024-11-14T09:49:49,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:49,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:49,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:49,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:49,935 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 061095f6a45343c99f344a39e72e7a5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:49,935 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:49,935 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:49,935 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,935 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/50a149ef786d4d5cb11a4b42bb77d17e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=35.4 K 2024-11-14T09:49:49,936 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c51a32aa48a54f00b50e9c82221be0d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731577787749 2024-11-14T09:49:49,936 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a149ef786d4d5cb11a4b42bb77d17e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:49,936 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c87386a005403c83a94cfbf250ca7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731577787749 2024-11-14T09:49:49,936 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239b037afda74e86a3cc3b834635d206, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:49,937 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ad4ee91716a4faabb8fb319446e4ca5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:49,947 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:49,947 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/db52b8ffc9204b7e8907cfb07502767f is 50, key is test_row_0/A:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,950 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:49,950 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/cffb7f6ac906432ba2807418a527b3c6 is 50, key is test_row_0/B:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742281_1457 (size=12359) 2024-11-14T09:49:49,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742282_1458 (size=12359) 2024-11-14T09:49:49,970 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/cffb7f6ac906432ba2807418a527b3c6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cffb7f6ac906432ba2807418a527b3c6 2024-11-14T09:49:49,972 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/db52b8ffc9204b7e8907cfb07502767f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/db52b8ffc9204b7e8907cfb07502767f 2024-11-14T09:49:49,975 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into cffb7f6ac906432ba2807418a527b3c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:49,975 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,975 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577789934; duration=0sec 2024-11-14T09:49:49,975 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:49,975 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:49,975 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:49,977 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into db52b8ffc9204b7e8907cfb07502767f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:49,977 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577789933; duration=0sec 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:49,977 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:49,977 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/86dbb1e290354c85ab4af032d2fc9aab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=35.4 K 2024-11-14T09:49:49,977 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dbb1e290354c85ab4af032d2fc9aab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731577787072 2024-11-14T09:49:49,978 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f9b0306ba84cd0b2ce375f7acf8155, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731577787749 2024-11-14T09:49:49,980 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ab2d700fb5436483e44a91bb154b49, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:49,986 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:49,987 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/6fadcae5242e45f08802b45327dd22d9 is 50, key is test_row_0/C:col10/1731577789080/Put/seqid=0 2024-11-14T09:49:49,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742283_1459 (size=12359) 2024-11-14T09:49:50,013 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/6fadcae5242e45f08802b45327dd22d9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6fadcae5242e45f08802b45327dd22d9 2024-11-14T09:49:50,023 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 6fadcae5242e45f08802b45327dd22d9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:50,023 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:50,023 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577789935; duration=0sec 2024-11-14T09:49:50,023 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:50,023 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:50,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T09:49:50,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:50,061 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T09:49:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/85c93aff399f4243bb033b4478972ccf is 50, key is test_row_0/A:col10/1731577789143/Put/seqid=0 2024-11-14T09:49:50,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742284_1460 (size=12151) 2024-11-14T09:49:50,072 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/85c93aff399f4243bb033b4478972ccf 2024-11-14T09:49:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:49:50,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/03cd0e1404d34f1980bc31391bdc8a1e is 50, key is test_row_0/B:col10/1731577789143/Put/seqid=0 2024-11-14T09:49:50,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742285_1461 (size=12151) 2024-11-14T09:49:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:50,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:50,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577850166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577850167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577850285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577850285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577850295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577850314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577850315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,486 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/03cd0e1404d34f1980bc31391bdc8a1e 2024-11-14T09:49:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c8d4c78f8ae741b5b2454cac819afe86 is 50, key is test_row_0/C:col10/1731577789143/Put/seqid=0 2024-11-14T09:49:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577850495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577850495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742286_1462 (size=12151) 2024-11-14T09:49:50,502 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c8d4c78f8ae741b5b2454cac819afe86 2024-11-14T09:49:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/85c93aff399f4243bb033b4478972ccf as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf 2024-11-14T09:49:50,510 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T09:49:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/03cd0e1404d34f1980bc31391bdc8a1e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e 2024-11-14T09:49:50,514 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T09:49:50,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c8d4c78f8ae741b5b2454cac819afe86 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86 2024-11-14T09:49:50,518 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T09:49:50,519 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 76f327b9ee0a90edb9e6efb3d43a25dd in 458ms, sequenceid=155, compaction requested=false 2024-11-14T09:49:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:50,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T09:49:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T09:49:50,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T09:49:50,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0720 sec 2024-11-14T09:49:50,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0800 sec 2024-11-14T09:49:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T09:49:50,585 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:50,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T09:49:50,589 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:50,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:50,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:50,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T09:49:50,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:50,742 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:49:50,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/5124c7c148bd4b7c9c2753849de3ed06 is 50, key is test_row_0/A:col10/1731577790166/Put/seqid=0 2024-11-14T09:49:50,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742287_1463 (size=12151) 2024-11-14T09:49:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:50,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577850903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577850904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:50,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577851014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577851014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,155 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/5124c7c148bd4b7c9c2753849de3ed06 2024-11-14T09:49:51,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d is 50, key is test_row_0/B:col10/1731577790166/Put/seqid=0 2024-11-14T09:49:51,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742288_1464 (size=12151) 2024-11-14T09:49:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577851225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577851225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577851315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,316 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577851325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,328 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577851325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,329 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577851535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577851535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:51,567 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d 2024-11-14T09:49:51,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/631659ae4c9542de80ed4e9d69027c67 is 50, key is test_row_0/C:col10/1731577790166/Put/seqid=0 2024-11-14T09:49:51,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742289_1465 (size=12151) 2024-11-14T09:49:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:51,989 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/631659ae4c9542de80ed4e9d69027c67 2024-11-14T09:49:51,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/5124c7c148bd4b7c9c2753849de3ed06 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06 2024-11-14T09:49:51,998 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T09:49:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d 2024-11-14T09:49:52,002 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T09:49:52,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/631659ae4c9542de80ed4e9d69027c67 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67 2024-11-14T09:49:52,007 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67, entries=150, sequenceid=171, filesize=11.9 K 2024-11-14T09:49:52,008 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1265ms, sequenceid=171, compaction requested=true 2024-11-14T09:49:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T09:49:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T09:49:52,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T09:49:52,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-14T09:49:52,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.4250 sec 2024-11-14T09:49:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:52,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:49:52,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/966c7f2ad1ea4bf89f45a4dd248452be is 50, key is test_row_0/A:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:52,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742290_1466 (size=14541) 2024-11-14T09:49:52,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577852109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577852116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577852225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577852225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577852435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577852435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/966c7f2ad1ea4bf89f45a4dd248452be 2024-11-14T09:49:52,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/a06a17c5829b46b5aed462e561491aa4 is 50, key is test_row_0/B:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:52,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742291_1467 (size=12151) 2024-11-14T09:49:52,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/a06a17c5829b46b5aed462e561491aa4 2024-11-14T09:49:52,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/6c909e4cf3a54565a2c4880121ea679f is 50, key is test_row_0/C:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:52,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742292_1468 (size=12151) 2024-11-14T09:49:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T09:49:52,735 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:52,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T09:49:52,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:52,737 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:52,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:49:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577852755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577852755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:49:52,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:49:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:52,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:52,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:52,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/6c909e4cf3a54565a2c4880121ea679f 2024-11-14T09:49:52,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/966c7f2ad1ea4bf89f45a4dd248452be as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be 2024-11-14T09:49:52,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be, entries=200, sequenceid=192, filesize=14.2 K 2024-11-14T09:49:52,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/a06a17c5829b46b5aed462e561491aa4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4 2024-11-14T09:49:52,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T09:49:52,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/6c909e4cf3a54565a2c4880121ea679f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f 2024-11-14T09:49:52,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f, entries=150, sequenceid=192, filesize=11.9 K 2024-11-14T09:49:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 76f327b9ee0a90edb9e6efb3d43a25dd in 898ms, sequenceid=192, compaction requested=true 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:52,959 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:49:52,959 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:52,960 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51202 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:52,960 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:52,960 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:52,960 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/db52b8ffc9204b7e8907cfb07502767f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=50.0 K 2024-11-14T09:49:52,961 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:52,961 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:52,961 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:52,961 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cffb7f6ac906432ba2807418a527b3c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=47.7 K 2024-11-14T09:49:52,961 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting db52b8ffc9204b7e8907cfb07502767f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:52,962 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffb7f6ac906432ba2807418a527b3c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:52,962 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03cd0e1404d34f1980bc31391bdc8a1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577789131 2024-11-14T09:49:52,962 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 85c93aff399f4243bb033b4478972ccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577789131 2024-11-14T09:49:52,965 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7e529f0c1c4bcab3ff10cd2ac6ee8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731577790138 2024-11-14T09:49:52,965 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5124c7c148bd4b7c9c2753849de3ed06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731577790138 2024-11-14T09:49:52,965 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06a17c5829b46b5aed462e561491aa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:52,965 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 966c7f2ad1ea4bf89f45a4dd248452be, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:52,982 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:52,983 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/cd9cbe2aeee2400387ea4e305cb0dc51 is 50, key is test_row_0/B:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:52,985 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:52,985 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/490177fc2f734d768bc25ab937d9e944 is 50, key is test_row_0/A:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742293_1469 (size=12595) 2024-11-14T09:49:52,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742294_1470 (size=12595) 2024-11-14T09:49:53,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/490177fc2f734d768bc25ab937d9e944 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/490177fc2f734d768bc25ab937d9e944 2024-11-14T09:49:53,012 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/cd9cbe2aeee2400387ea4e305cb0dc51 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cd9cbe2aeee2400387ea4e305cb0dc51 2024-11-14T09:49:53,013 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 490177fc2f734d768bc25ab937d9e944(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:53,013 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:53,013 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=12, startTime=1731577792959; duration=0sec 2024-11-14T09:49:53,013 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:53,013 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:53,013 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:49:53,015 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:49:53,015 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:53,015 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:53,015 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6fadcae5242e45f08802b45327dd22d9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=47.7 K 2024-11-14T09:49:53,015 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fadcae5242e45f08802b45327dd22d9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731577788942 2024-11-14T09:49:53,016 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d4c78f8ae741b5b2454cac819afe86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731577789131 2024-11-14T09:49:53,016 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 631659ae4c9542de80ed4e9d69027c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731577790138 2024-11-14T09:49:53,016 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c909e4cf3a54565a2c4880121ea679f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:53,018 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into cd9cbe2aeee2400387ea4e305cb0dc51(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:53,018 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:53,018 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=12, startTime=1731577792959; duration=0sec 2024-11-14T09:49:53,018 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:53,018 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:53,024 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:53,025 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/0315bd3679ff4abc90533e9bc2be2119 is 50, key is test_row_0/C:col10/1731577790902/Put/seqid=0 2024-11-14T09:49:53,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T09:49:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:53,042 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/60850a81e8b445adb2ee8c9dc1c96711 is 50, key is test_row_0/A:col10/1731577792114/Put/seqid=0 2024-11-14T09:49:53,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:49:53,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742295_1471 (size=12595) 2024-11-14T09:49:53,059 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/0315bd3679ff4abc90533e9bc2be2119 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0315bd3679ff4abc90533e9bc2be2119 2024-11-14T09:49:53,064 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 0315bd3679ff4abc90533e9bc2be2119(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:53,065 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:53,065 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=12, startTime=1731577792959; duration=0sec 2024-11-14T09:49:53,065 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:53,065 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:53,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742296_1472 (size=12151) 2024-11-14T09:49:53,071 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/60850a81e8b445adb2ee8c9dc1c96711 2024-11-14T09:49:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/1013844f7c844f709f3e8e01a57f9ad0 is 50, key is test_row_0/B:col10/1731577792114/Put/seqid=0 2024-11-14T09:49:53,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742297_1473 (size=12151) 2024-11-14T09:49:53,085 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/1013844f7c844f709f3e8e01a57f9ad0 2024-11-14T09:49:53,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/8be6f821289b40edae55f2937b0c5d31 is 50, key is test_row_0/C:col10/1731577792114/Put/seqid=0 2024-11-14T09:49:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742298_1474 (size=12151) 2024-11-14T09:49:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:53,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577853341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577853343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577853345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577853345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577853345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,354 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,355 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,355 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:49:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577853465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577853465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,512 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/8be6f821289b40edae55f2937b0c5d31 2024-11-14T09:49:53,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/60850a81e8b445adb2ee8c9dc1c96711 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711 2024-11-14T09:49:53,522 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711, entries=150, sequenceid=209, filesize=11.9 K 2024-11-14T09:49:53,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/1013844f7c844f709f3e8e01a57f9ad0 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0 2024-11-14T09:49:53,527 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0, entries=150, sequenceid=209, filesize=11.9 K 2024-11-14T09:49:53,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/8be6f821289b40edae55f2937b0c5d31 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31 2024-11-14T09:49:53,532 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31, entries=150, sequenceid=209, filesize=11.9 K 2024-11-14T09:49:53,532 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 76f327b9ee0a90edb9e6efb3d43a25dd in 490ms, sequenceid=209, compaction requested=false 2024-11-14T09:49:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T09:49:53,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T09:49:53,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T09:49:53,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 797 msec 2024-11-14T09:49:53,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 801 msec 2024-11-14T09:49:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:53,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:53,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/b5429074cdf14dadad8431d65f6efac2 is 50, key is test_row_0/A:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:53,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742299_1475 (size=14541) 2024-11-14T09:49:53,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577853718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577853720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577853835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577853835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T09:49:53,875 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:53,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T09:49:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:53,877 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:53,877 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:53,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:54,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577854045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577854045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/b5429074cdf14dadad8431d65f6efac2 2024-11-14T09:49:54,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7f586f7891334db883954c2f3800106b is 50, key is test_row_0/B:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742300_1476 (size=12151) 2024-11-14T09:49:54,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:54,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577854354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577854355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7f586f7891334db883954c2f3800106b 2024-11-14T09:49:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:54,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/5fd43e9e64434465919d70a4b336890f is 50, key is test_row_0/C:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:54,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742301_1477 (size=12151) 2024-11-14T09:49:54,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:54,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:54,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577854865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577854865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:54,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/5fd43e9e64434465919d70a4b336890f 2024-11-14T09:49:54,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/b5429074cdf14dadad8431d65f6efac2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2 2024-11-14T09:49:54,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2, entries=200, sequenceid=232, filesize=14.2 K 2024-11-14T09:49:54,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7f586f7891334db883954c2f3800106b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b 2024-11-14T09:49:54,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b, entries=150, sequenceid=232, filesize=11.9 K 2024-11-14T09:49:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/5fd43e9e64434465919d70a4b336890f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f 2024-11-14T09:49:54,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f, entries=150, sequenceid=232, filesize=11.9 K 2024-11-14T09:49:54,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1254ms, sequenceid=232, compaction requested=true 2024-11-14T09:49:54,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:54,933 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:54,933 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:54,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:54,933 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:54,934 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:54,934 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,934 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/490177fc2f734d768bc25ab937d9e944, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=38.4 K 2024-11-14T09:49:54,934 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:54,934 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:54,934 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,934 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cd9cbe2aeee2400387ea4e305cb0dc51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.0 K 2024-11-14T09:49:54,934 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9cbe2aeee2400387ea4e305cb0dc51, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:54,934 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 490177fc2f734d768bc25ab937d9e944, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:54,935 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1013844f7c844f709f3e8e01a57f9ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731577792104 2024-11-14T09:49:54,935 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60850a81e8b445adb2ee8c9dc1c96711, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731577792104 2024-11-14T09:49:54,935 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5429074cdf14dadad8431d65f6efac2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:54,935 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f586f7891334db883954c2f3800106b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:54,942 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:54,942 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:54,942 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e1670e3a22844be2ab713e7bbf7adaf5 is 50, key is test_row_0/A:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:54,942 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/e7bebdfd9e8a424991b85a44e68cc540 is 50, key is test_row_0/B:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:54,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T09:49:54,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:54,947 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:49:54,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742303_1479 (size=12697) 2024-11-14T09:49:54,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742302_1478 (size=12697) 2024-11-14T09:49:54,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/761e6cf76a714a108d25c756cb5e0ac2 is 50, key is test_row_0/A:col10/1731577793678/Put/seqid=0 2024-11-14T09:49:54,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742304_1480 (size=12151) 2024-11-14T09:49:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:55,362 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/761e6cf76a714a108d25c756cb5e0ac2 2024-11-14T09:49:55,363 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e1670e3a22844be2ab713e7bbf7adaf5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1670e3a22844be2ab713e7bbf7adaf5 2024-11-14T09:49:55,366 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/e7bebdfd9e8a424991b85a44e68cc540 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e7bebdfd9e8a424991b85a44e68cc540 2024-11-14T09:49:55,368 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into e1670e3a22844be2ab713e7bbf7adaf5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:55,368 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:55,368 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577794933; duration=0sec 2024-11-14T09:49:55,368 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:55,368 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:55,368 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:55,369 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:55,369 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:55,369 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:55,369 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0315bd3679ff4abc90533e9bc2be2119, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.0 K 2024-11-14T09:49:55,370 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0315bd3679ff4abc90533e9bc2be2119, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731577790859 2024-11-14T09:49:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/e78b63ad99204f0fb0d72d8c3a604632 is 50, key is test_row_0/B:col10/1731577793678/Put/seqid=0 2024-11-14T09:49:55,370 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be6f821289b40edae55f2937b0c5d31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731577792104 2024-11-14T09:49:55,370 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd43e9e64434465919d70a4b336890f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:55,373 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into e7bebdfd9e8a424991b85a44e68cc540(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:55,373 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:55,373 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577794933; duration=0sec 2024-11-14T09:49:55,373 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:55,373 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:55,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742305_1481 (size=12151) 2024-11-14T09:49:55,375 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/e78b63ad99204f0fb0d72d8c3a604632 2024-11-14T09:49:55,390 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:55,391 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/d97348b84fd244c6aa4b48cdb9da506c is 50, key is test_row_0/C:col10/1731577793310/Put/seqid=0 2024-11-14T09:49:55,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/cf51d9ae631b488ca08806817df02529 is 50, key is test_row_0/C:col10/1731577793678/Put/seqid=0 2024-11-14T09:49:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742306_1482 (size=12697) 2024-11-14T09:49:55,415 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/d97348b84fd244c6aa4b48cdb9da506c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/d97348b84fd244c6aa4b48cdb9da506c 2024-11-14T09:49:55,422 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into d97348b84fd244c6aa4b48cdb9da506c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:55,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:55,422 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577794933; duration=0sec 2024-11-14T09:49:55,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:55,422 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:55,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742307_1483 (size=12151) 2024-11-14T09:49:55,831 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/cf51d9ae631b488ca08806817df02529 2024-11-14T09:49:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/761e6cf76a714a108d25c756cb5e0ac2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2 2024-11-14T09:49:55,840 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T09:49:55,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/e78b63ad99204f0fb0d72d8c3a604632 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632 2024-11-14T09:49:55,844 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T09:49:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/cf51d9ae631b488ca08806817df02529 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529 2024-11-14T09:49:55,849 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T09:49:55,850 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 76f327b9ee0a90edb9e6efb3d43a25dd in 903ms, sequenceid=246, compaction requested=false 2024-11-14T09:49:55,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:55,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:55,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T09:49:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T09:49:55,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T09:49:55,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-14T09:49:55,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9780 sec 2024-11-14T09:49:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:55,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:55,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:49:55,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e9c5a5c06e5b4dceba5813275492b3c9 is 50, key is test_row_0/A:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742308_1484 (size=14691) 2024-11-14T09:49:55,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e9c5a5c06e5b4dceba5813275492b3c9 2024-11-14T09:49:55,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/35086dfc0e1742d699c719b8f3fc73e7 is 50, key is test_row_0/B:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:55,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742309_1485 (size=12251) 2024-11-14T09:49:56,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577855995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577856000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T09:49:56,025 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:56,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T09:49:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:56,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:56,028 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:56,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577856125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577856125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:56,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:49:56,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:56,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/35086dfc0e1742d699c719b8f3fc73e7 2024-11-14T09:49:56,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c56d1d0a4e71444a99c2e191948e2d0f is 50, key is test_row_0/C:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:56,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:49:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:56,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742310_1486 (size=12251) 2024-11-14T09:49:56,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577856346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577856346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:49:56,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:56,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:49:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:56,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577856665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577856665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:56,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c56d1d0a4e71444a99c2e191948e2d0f 2024-11-14T09:49:56,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e9c5a5c06e5b4dceba5813275492b3c9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9 2024-11-14T09:49:56,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9, entries=200, sequenceid=260, filesize=14.3 K 2024-11-14T09:49:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/35086dfc0e1742d699c719b8f3fc73e7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7 2024-11-14T09:49:56,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7, entries=150, sequenceid=260, filesize=12.0 K 2024-11-14T09:49:56,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c56d1d0a4e71444a99c2e191948e2d0f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f 2024-11-14T09:49:56,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f, entries=150, sequenceid=260, filesize=12.0 K 2024-11-14T09:49:56,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 76f327b9ee0a90edb9e6efb3d43a25dd in 871ms, sequenceid=260, compaction requested=true 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:56,766 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:56,766 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:56,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:56,767 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,767 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,767 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1670e3a22844be2ab713e7bbf7adaf5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=38.6 K 2024-11-14T09:49:56,767 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e7bebdfd9e8a424991b85a44e68cc540, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.2 K 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1670e3a22844be2ab713e7bbf7adaf5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:56,767 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e7bebdfd9e8a424991b85a44e68cc540, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:56,768 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761e6cf76a714a108d25c756cb5e0ac2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731577793678 2024-11-14T09:49:56,768 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e78b63ad99204f0fb0d72d8c3a604632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731577793678 2024-11-14T09:49:56,768 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 35086dfc0e1742d699c719b8f3fc73e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:56,768 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9c5a5c06e5b4dceba5813275492b3c9, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:56,773 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:56,774 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/d11fabea106a454184078defecc539f2 is 50, key is test_row_0/B:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:56,775 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:56,775 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/23279af0df0e4734ae9749b93a22f448 is 50, key is test_row_0/A:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:56,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742311_1487 (size=12899) 2024-11-14T09:49:56,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742312_1488 (size=12899) 2024-11-14T09:49:56,791 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/23279af0df0e4734ae9749b93a22f448 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/23279af0df0e4734ae9749b93a22f448 2024-11-14T09:49:56,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T09:49:56,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,794 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:49:56,795 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 23279af0df0e4734ae9749b93a22f448(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:56,795 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:56,795 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577796766; duration=0sec 2024-11-14T09:49:56,795 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:56,795 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:56,795 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:56,796 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:56,796 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:56,796 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:56,796 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/d97348b84fd244c6aa4b48cdb9da506c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.2 K 2024-11-14T09:49:56,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/abebd4fc3d2a4d13816549162eecce35 is 50, key is test_row_0/A:col10/1731577795991/Put/seqid=0 2024-11-14T09:49:56,799 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting d97348b84fd244c6aa4b48cdb9da506c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731577793310 2024-11-14T09:49:56,800 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf51d9ae631b488ca08806817df02529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731577793678 2024-11-14T09:49:56,800 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56d1d0a4e71444a99c2e191948e2d0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:56,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742313_1489 (size=12301) 2024-11-14T09:49:56,805 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:56,805 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/76589559678042e6ae5094b8e7e4d23b is 50, key is test_row_0/C:col10/1731577795895/Put/seqid=0 2024-11-14T09:49:56,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742314_1490 (size=12899) 2024-11-14T09:49:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:57,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:57,182 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/d11fabea106a454184078defecc539f2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/d11fabea106a454184078defecc539f2 2024-11-14T09:49:57,185 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into d11fabea106a454184078defecc539f2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:57,185 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:57,185 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577796766; duration=0sec 2024-11-14T09:49:57,186 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:57,186 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:57,201 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/abebd4fc3d2a4d13816549162eecce35 2024-11-14T09:49:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/154c946303ce41959a0c633c4c7797e3 is 50, key is test_row_0/B:col10/1731577795991/Put/seqid=0 2024-11-14T09:49:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742315_1491 (size=12301) 2024-11-14T09:49:57,215 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/154c946303ce41959a0c633c4c7797e3 2024-11-14T09:49:57,217 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/76589559678042e6ae5094b8e7e4d23b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/76589559678042e6ae5094b8e7e4d23b 2024-11-14T09:49:57,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/61984e7e673b46518dbb132d33a88cf8 is 50, key is test_row_0/C:col10/1731577795991/Put/seqid=0 2024-11-14T09:49:57,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742316_1492 (size=12301) 2024-11-14T09:49:57,235 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 76589559678042e6ae5094b8e7e4d23b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:57,235 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:57,235 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577796766; duration=0sec 2024-11-14T09:49:57,235 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:57,235 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577857235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577857235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577857354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577857354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54204 deadline: 1731577857375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54172 deadline: 1731577857375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,381 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:57,381 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54238 deadline: 1731577857395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,400 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_2' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:49:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577857564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577857564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,624 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/61984e7e673b46518dbb132d33a88cf8 2024-11-14T09:49:57,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/abebd4fc3d2a4d13816549162eecce35 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35 2024-11-14T09:49:57,630 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T09:49:57,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/154c946303ce41959a0c633c4c7797e3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3 2024-11-14T09:49:57,634 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T09:49:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/61984e7e673b46518dbb132d33a88cf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8 2024-11-14T09:49:57,638 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T09:49:57,638 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 76f327b9ee0a90edb9e6efb3d43a25dd in 844ms, sequenceid=285, compaction requested=false 2024-11-14T09:49:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T09:49:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T09:49:57,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T09:49:57,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6120 sec 2024-11-14T09:49:57,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6240 sec 2024-11-14T09:49:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:57,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:57,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T09:49:57,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f1995166ab7a47e389fdbbec91f20e4e is 50, key is test_row_0/A:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:57,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742317_1493 (size=17181) 2024-11-14T09:49:57,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f1995166ab7a47e389fdbbec91f20e4e 2024-11-14T09:49:57,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/ed146ac18c07422088cff6b1df1762eb is 50, key is test_row_0/B:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:57,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742318_1494 (size=12301) 2024-11-14T09:49:57,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/ed146ac18c07422088cff6b1df1762eb 2024-11-14T09:49:57,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/93a8f55410a34917a961b635a26cd8c9 is 50, key is test_row_0/C:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:57,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742319_1495 (size=12301) 2024-11-14T09:49:57,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577857943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577857951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577858054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577858065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T09:49:58,175 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:58,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T09:49:58,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:49:58,176 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:58,177 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:58,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:58,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577858269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577858274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:49:58,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/93a8f55410a34917a961b635a26cd8c9 2024-11-14T09:49:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/f1995166ab7a47e389fdbbec91f20e4e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e 2024-11-14T09:49:58,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e, entries=250, sequenceid=302, filesize=16.8 K 2024-11-14T09:49:58,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/ed146ac18c07422088cff6b1df1762eb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb 2024-11-14T09:49:58,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T09:49:58,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/93a8f55410a34917a961b635a26cd8c9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9 2024-11-14T09:49:58,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9, entries=150, sequenceid=302, filesize=12.0 K 2024-11-14T09:49:58,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 76f327b9ee0a90edb9e6efb3d43a25dd in 440ms, sequenceid=302, compaction requested=true 2024-11-14T09:49:58,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:58,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:49:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:49:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:49:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:49:58,322 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:49:58,322 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:58,322 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:58,322 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/d11fabea106a454184078defecc539f2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.6 K 2024-11-14T09:49:58,322 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/23279af0df0e4734ae9749b93a22f448, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=41.4 K 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23279af0df0e4734ae9749b93a22f448, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d11fabea106a454184078defecc539f2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 154c946303ce41959a0c633c4c7797e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731577795973 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting abebd4fc3d2a4d13816549162eecce35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731577795973 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ed146ac18c07422088cff6b1df1762eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:49:58,323 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1995166ab7a47e389fdbbec91f20e4e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:49:58,328 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:58,328 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:58,329 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/ece00ce9585148d1b5b088462d669c76 is 50, key is test_row_0/A:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:58,329 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/09b02c05c673459ca1b6ebc9cf8ef40c is 50, key is test_row_0/B:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:58,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T09:49:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:58,330 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T09:49:58,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742321_1497 (size=13051) 2024-11-14T09:49:58,337 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/09b02c05c673459ca1b6ebc9cf8ef40c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09b02c05c673459ca1b6ebc9cf8ef40c 2024-11-14T09:49:58,341 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 09b02c05c673459ca1b6ebc9cf8ef40c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:58,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:58,341 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577798322; duration=0sec 2024-11-14T09:49:58,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:49:58,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:49:58,341 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:49:58,342 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:49:58,342 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:49:58,342 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:58,342 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/76589559678042e6ae5094b8e7e4d23b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.6 K 2024-11-14T09:49:58,343 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 76589559678042e6ae5094b8e7e4d23b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731577795883 2024-11-14T09:49:58,343 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 61984e7e673b46518dbb132d33a88cf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731577795973 2024-11-14T09:49:58,343 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a8f55410a34917a961b635a26cd8c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:49:58,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742320_1496 (size=13051) 2024-11-14T09:49:58,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/987691057f144805a2db3eaba2ea900d is 50, key is test_row_0/A:col10/1731577797949/Put/seqid=0 2024-11-14T09:49:58,361 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:49:58,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742322_1498 (size=12301) 2024-11-14T09:49:58,362 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/8f7c37078f0b43148010df82f510ab6f is 50, key is test_row_0/C:col10/1731577797879/Put/seqid=0 2024-11-14T09:49:58,362 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/987691057f144805a2db3eaba2ea900d 2024-11-14T09:49:58,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742323_1499 (size=13051) 2024-11-14T09:49:58,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/9dd2d8b293d74332b06dd0e30e752a9b is 50, key is test_row_0/B:col10/1731577797949/Put/seqid=0 2024-11-14T09:49:58,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742324_1500 (size=12301) 2024-11-14T09:49:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:49:58,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:58,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577858631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577858632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,749 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/ece00ce9585148d1b5b088462d669c76 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/ece00ce9585148d1b5b088462d669c76 2024-11-14T09:49:58,752 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into ece00ce9585148d1b5b088462d669c76(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:58,752 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:58,752 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577798321; duration=0sec 2024-11-14T09:49:58,752 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:58,752 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:49:58,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577858744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577858745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,775 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/8f7c37078f0b43148010df82f510ab6f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8f7c37078f0b43148010df82f510ab6f 2024-11-14T09:49:58,779 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 8f7c37078f0b43148010df82f510ab6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:49:58,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:58,779 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577798322; duration=0sec 2024-11-14T09:49:58,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:49:58,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:49:58,782 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/9dd2d8b293d74332b06dd0e30e752a9b 2024-11-14T09:49:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b4cdcbc5342649bca58d79e2bf733b44 is 50, key is test_row_0/C:col10/1731577797949/Put/seqid=0 2024-11-14T09:49:58,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742325_1501 (size=12301) 2024-11-14T09:49:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:49:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577858964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577858965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,201 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b4cdcbc5342649bca58d79e2bf733b44 2024-11-14T09:49:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/987691057f144805a2db3eaba2ea900d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d 2024-11-14T09:49:59,209 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T09:49:59,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/9dd2d8b293d74332b06dd0e30e752a9b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b 2024-11-14T09:49:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T09:49:59,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/b4cdcbc5342649bca58d79e2bf733b44 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44 2024-11-14T09:49:59,220 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T09:49:59,221 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 76f327b9ee0a90edb9e6efb3d43a25dd in 891ms, sequenceid=324, compaction requested=false 2024-11-14T09:49:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:49:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T09:49:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T09:49:59,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T09:49:59,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0450 sec 2024-11-14T09:49:59,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0490 sec 2024-11-14T09:49:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:49:59,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:49:59,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:49:59,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cffe52a33db84563af2bf96a77403067 is 50, key is test_row_0/A:col10/1731577799277/Put/seqid=0 2024-11-14T09:49:59,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742326_1502 (size=14741) 2024-11-14T09:49:59,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cffe52a33db84563af2bf96a77403067 2024-11-14T09:49:59,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7e5a0a24e7a646189389d303d1148e4b is 50, key is test_row_0/B:col10/1731577799277/Put/seqid=0 2024-11-14T09:49:59,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742327_1503 (size=12301) 2024-11-14T09:49:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T09:49:59,315 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:49:59,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:49:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T09:49:59,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:49:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:49:59,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:49:59,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:49:59,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577859339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577859346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:49:59,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577859454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:49:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577859464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:49:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:49:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577859664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577859675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7e5a0a24e7a646189389d303d1148e4b 2024-11-14T09:49:59,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c3a6dbe0370f45f2843655309bf1d513 is 50, key is test_row_0/C:col10/1731577799277/Put/seqid=0 2024-11-14T09:49:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742328_1504 (size=12301) 2024-11-14T09:49:59,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:49:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:49:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:49:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:49:59,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:49:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:49:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577859975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:49:59,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:49:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577859984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:49:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:49:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:50:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:00,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:00,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c3a6dbe0370f45f2843655309bf1d513 2024-11-14T09:50:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cffe52a33db84563af2bf96a77403067 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067 2024-11-14T09:50:00,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067, entries=200, sequenceid=342, filesize=14.4 K 2024-11-14T09:50:00,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/7e5a0a24e7a646189389d303d1148e4b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b 2024-11-14T09:50:00,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:50:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/c3a6dbe0370f45f2843655309bf1d513 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513 2024-11-14T09:50:00,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T09:50:00,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 76f327b9ee0a90edb9e6efb3d43a25dd in 848ms, sequenceid=342, compaction requested=true 2024-11-14T09:50:00,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:00,127 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:00,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:00,127 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:50:00,128 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,128 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/ece00ce9585148d1b5b088462d669c76, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=39.2 K 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:50:00,128 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,128 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09b02c05c673459ca1b6ebc9cf8ef40c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.8 K 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ece00ce9585148d1b5b088462d669c76, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:50:00,128 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b02c05c673459ca1b6ebc9cf8ef40c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:50:00,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987691057f144805a2db3eaba2ea900d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577797940 2024-11-14T09:50:00,129 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dd2d8b293d74332b06dd0e30e752a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577797940 2024-11-14T09:50:00,129 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cffe52a33db84563af2bf96a77403067, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798605 2024-11-14T09:50:00,129 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5a0a24e7a646189389d303d1148e4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798629 2024-11-14T09:50:00,134 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:00,134 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/8c9b9221fedd4f7291363915ceb13502 is 50, key is test_row_0/B:col10/1731577799277/Put/seqid=0 2024-11-14T09:50:00,134 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:00,135 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/46cc4ea6b4364c61af647a6c04806cce is 50, key is test_row_0/A:col10/1731577799277/Put/seqid=0 2024-11-14T09:50:00,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742329_1505 (size=13153) 2024-11-14T09:50:00,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742330_1506 (size=13153) 2024-11-14T09:50:00,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T09:50:00,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,235 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:50:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e1fb6ab85b31417db3f646d64f78e15a is 50, key is test_row_0/A:col10/1731577799344/Put/seqid=0 2024-11-14T09:50:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742331_1507 (size=12301) 2024-11-14T09:50:00,242 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e1fb6ab85b31417db3f646d64f78e15a 2024-11-14T09:50:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c2341823fab64a739656509eeec96820 is 50, key is test_row_0/B:col10/1731577799344/Put/seqid=0 2024-11-14T09:50:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742332_1508 (size=12301) 2024-11-14T09:50:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:50:00,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:00,545 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/46cc4ea6b4364c61af647a6c04806cce as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/46cc4ea6b4364c61af647a6c04806cce 2024-11-14T09:50:00,548 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 46cc4ea6b4364c61af647a6c04806cce(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:00,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:00,548 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577800126; duration=0sec 2024-11-14T09:50:00,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:00,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:50:00,548 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:00,549 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:00,549 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:50:00,549 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,549 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8f7c37078f0b43148010df82f510ab6f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.8 K 2024-11-14T09:50:00,549 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f7c37078f0b43148010df82f510ab6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731577797206 2024-11-14T09:50:00,550 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4cdcbc5342649bca58d79e2bf733b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577797940 2024-11-14T09:50:00,550 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a6dbe0370f45f2843655309bf1d513, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798629 2024-11-14T09:50:00,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577860552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,555 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/8c9b9221fedd4f7291363915ceb13502 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c9b9221fedd4f7291363915ceb13502 2024-11-14T09:50:00,557 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:00,557 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/9db1f7b720a04c7f830998ac4264ce59 is 50, key is test_row_0/C:col10/1731577799277/Put/seqid=0 2024-11-14T09:50:00,559 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 8c9b9221fedd4f7291363915ceb13502(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:00,559 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:00,559 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577800127; duration=0sec 2024-11-14T09:50:00,559 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:00,559 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:50:00,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742333_1509 (size=13153) 2024-11-14T09:50:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577860553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,651 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c2341823fab64a739656509eeec96820 2024-11-14T09:50:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/87a96f5a232742e9af6d4e6345ff8a27 is 50, key is test_row_0/C:col10/1731577799344/Put/seqid=0 2024-11-14T09:50:00,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742334_1510 (size=12301) 2024-11-14T09:50:00,659 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/87a96f5a232742e9af6d4e6345ff8a27 2024-11-14T09:50:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e1fb6ab85b31417db3f646d64f78e15a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a 2024-11-14T09:50:00,665 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-14T09:50:00,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/c2341823fab64a739656509eeec96820 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820 2024-11-14T09:50:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577860664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,668 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820, entries=150, sequenceid=363, filesize=12.0 K 2024-11-14T09:50:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/87a96f5a232742e9af6d4e6345ff8a27 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27 2024-11-14T09:50:00,671 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27, entries=150, sequenceid=363, filesize=12.0 K 2024-11-14T09:50:00,672 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 76f327b9ee0a90edb9e6efb3d43a25dd in 436ms, sequenceid=363, compaction requested=false 2024-11-14T09:50:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T09:50:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T09:50:00,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T09:50:00,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3550 sec 2024-11-14T09:50:00,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3590 sec 2024-11-14T09:50:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:00,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:00,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T09:50:00,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/932238241bf44f75aa3c3f847db8c4d3 is 50, key is test_row_0/A:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:00,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742335_1511 (size=14741) 2024-11-14T09:50:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577860796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577860874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577860915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:00,968 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/9db1f7b720a04c7f830998ac4264ce59 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/9db1f7b720a04c7f830998ac4264ce59 2024-11-14T09:50:00,973 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 9db1f7b720a04c7f830998ac4264ce59(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:00,973 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:00,973 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577800127; duration=0sec 2024-11-14T09:50:00,974 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:00,974 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:50:01,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/932238241bf44f75aa3c3f847db8c4d3 2024-11-14T09:50:01,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/8c53dc074687491ba7cdda72337d1f19 is 50, key is test_row_0/B:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:01,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742336_1512 (size=12301) 2024-11-14T09:50:01,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577861124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:01,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577861185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577861435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T09:50:01,476 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:01,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T09:50:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:01,479 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:01,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:01,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:01,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/8c53dc074687491ba7cdda72337d1f19 2024-11-14T09:50:01,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/1370a25627444b7588733247d751495c is 50, key is test_row_0/C:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:01,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742337_1513 (size=12301) 2024-11-14T09:50:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:01,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577861695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:01,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:01,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/1370a25627444b7588733247d751495c 2024-11-14T09:50:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/932238241bf44f75aa3c3f847db8c4d3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3 2024-11-14T09:50:01,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:01,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:01,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:01,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3, entries=200, sequenceid=382, filesize=14.4 K 2024-11-14T09:50:01,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/8c53dc074687491ba7cdda72337d1f19 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19 2024-11-14T09:50:01,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19, entries=150, sequenceid=382, filesize=12.0 K 2024-11-14T09:50:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/1370a25627444b7588733247d751495c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c 2024-11-14T09:50:01,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c, entries=150, sequenceid=382, filesize=12.0 K 2024-11-14T09:50:01,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1268ms, sequenceid=382, compaction requested=true 2024-11-14T09:50:01,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:01,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:01,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:01,950 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:01,950 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:01,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:01,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:01,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:01,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:01,951 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:01,951 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:50:01,951 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,951 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/46cc4ea6b4364c61af647a6c04806cce, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=39.3 K 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46cc4ea6b4364c61af647a6c04806cce, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798629 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:50:01,952 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:01,952 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c9b9221fedd4f7291363915ceb13502, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.9 K 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1fb6ab85b31417db3f646d64f78e15a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731577799328 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9b9221fedd4f7291363915ceb13502, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798629 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 932238241bf44f75aa3c3f847db8c4d3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:01,952 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c2341823fab64a739656509eeec96820, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731577799328 2024-11-14T09:50:01,956 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c53dc074687491ba7cdda72337d1f19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:01,961 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:01,961 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:01,961 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/0c4ff2712b1a4a8d890d52bd05546930 is 50, key is test_row_0/B:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:01,961 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e920246cb98541bc9124aa01f1dbce35 is 50, key is test_row_0/A:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:01,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:01,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T09:50:01,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/7f8ff07e386c464e8e1928aa2392bf61 is 50, key is test_row_0/A:col10/1731577801961/Put/seqid=0 2024-11-14T09:50:01,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742338_1514 (size=13255) 2024-11-14T09:50:01,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742339_1515 (size=13255) 2024-11-14T09:50:02,003 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/0c4ff2712b1a4a8d890d52bd05546930 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/0c4ff2712b1a4a8d890d52bd05546930 2024-11-14T09:50:02,008 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 0c4ff2712b1a4a8d890d52bd05546930(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:02,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:02,009 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=13, startTime=1731577801950; duration=0sec 2024-11-14T09:50:02,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:02,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:50:02,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:02,010 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:02,010 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:50:02,010 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,010 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/9db1f7b720a04c7f830998ac4264ce59, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=36.9 K 2024-11-14T09:50:02,011 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db1f7b720a04c7f830998ac4264ce59, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731577798629 2024-11-14T09:50:02,011 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a96f5a232742e9af6d4e6345ff8a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731577799328 2024-11-14T09:50:02,011 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1370a25627444b7588733247d751495c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742340_1516 (size=14741) 2024-11-14T09:50:02,016 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:02,016 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/7b25f387dbf240af80056bba771b8f24 is 50, key is test_row_0/C:col10/1731577800551/Put/seqid=0 2024-11-14T09:50:02,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742341_1517 (size=13255) 2024-11-14T09:50:02,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577862086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:02,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577862205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:02,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,384 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/e920246cb98541bc9124aa01f1dbce35 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e920246cb98541bc9124aa01f1dbce35 2024-11-14T09:50:02,387 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into e920246cb98541bc9124aa01f1dbce35(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:02,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:02,387 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=13, startTime=1731577801950; duration=0sec 2024-11-14T09:50:02,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:02,387 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:50:02,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/7f8ff07e386c464e8e1928aa2392bf61 2024-11-14T09:50:02,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/5770bc75aa9a495d963e33bf5cc0458f is 50, key is test_row_0/B:col10/1731577801961/Put/seqid=0 2024-11-14T09:50:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742342_1518 (size=12301) 2024-11-14T09:50:02,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/5770bc75aa9a495d963e33bf5cc0458f 2024-11-14T09:50:02,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577862425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:02,435 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/7b25f387dbf240af80056bba771b8f24 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7b25f387dbf240af80056bba771b8f24 2024-11-14T09:50:02,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/28b2f4d26f414953b5c729ac29c9f4ba is 50, key is test_row_0/C:col10/1731577801961/Put/seqid=0 2024-11-14T09:50:02,439 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into 7b25f387dbf240af80056bba771b8f24(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:02,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:02,440 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=13, startTime=1731577801951; duration=0sec 2024-11-14T09:50:02,440 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:02,440 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:50:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742343_1519 (size=12301) 2024-11-14T09:50:02,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:02,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:02,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54196 deadline: 1731577862705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:02,710 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in 76f327b9ee0a90edb9e6efb3d43a25dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:02,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577862745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:02,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:02,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/28b2f4d26f414953b5c729ac29c9f4ba 2024-11-14T09:50:02,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/7f8ff07e386c464e8e1928aa2392bf61 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61 2024-11-14T09:50:02,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61, entries=200, sequenceid=403, filesize=14.4 K 2024-11-14T09:50:02,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/5770bc75aa9a495d963e33bf5cc0458f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f 2024-11-14T09:50:02,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T09:50:02,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/28b2f4d26f414953b5c729ac29c9f4ba as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba 2024-11-14T09:50:02,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:02,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:02,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba, entries=150, sequenceid=403, filesize=12.0 K 2024-11-14T09:50:02,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 76f327b9ee0a90edb9e6efb3d43a25dd in 905ms, sequenceid=403, compaction requested=false 2024-11-14T09:50:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:03,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T09:50:03,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,018 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T09:50:03,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/921e2254d2db49989cf79d439d8f3431 is 50, key is test_row_0/A:col10/1731577802030/Put/seqid=0 2024-11-14T09:50:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742344_1520 (size=12301) 2024-11-14T09:50:03,078 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/921e2254d2db49989cf79d439d8f3431 2024-11-14T09:50:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/17eb0cd42164405ba598b86dd26b25c3 is 50, key is test_row_0/B:col10/1731577802030/Put/seqid=0 2024-11-14T09:50:03,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742345_1521 (size=12301) 2024-11-14T09:50:03,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:03,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:03,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:03,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577863412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:03,514 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/17eb0cd42164405ba598b86dd26b25c3 2024-11-14T09:50:03,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/64d1d670e0c44d0db279b5345edb099d is 50, key is test_row_0/C:col10/1731577802030/Put/seqid=0 2024-11-14T09:50:03,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577863534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:03,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742346_1522 (size=12301) 2024-11-14T09:50:03,549 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/64d1d670e0c44d0db279b5345edb099d 2024-11-14T09:50:03,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/921e2254d2db49989cf79d439d8f3431 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431 2024-11-14T09:50:03,558 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T09:50:03,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/17eb0cd42164405ba598b86dd26b25c3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3 2024-11-14T09:50:03,570 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T09:50:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/64d1d670e0c44d0db279b5345edb099d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d 2024-11-14T09:50:03,576 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d, entries=150, sequenceid=422, filesize=12.0 K 2024-11-14T09:50:03,577 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 76f327b9ee0a90edb9e6efb3d43a25dd in 559ms, sequenceid=422, compaction requested=true 2024-11-14T09:50:03,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:03,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T09:50:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T09:50:03,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T09:50:03,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1030 sec 2024-11-14T09:50:03,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.1150 sec 2024-11-14T09:50:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T09:50:03,625 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:03,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:03,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-14T09:50:03,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:03,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:03,628 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:03,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:03,754 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:03,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T09:50:03,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cd875e2b5ea94968a8c1a45e3cae2d6e is 50, key is test_row_0/A:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:03,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T09:50:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742347_1523 (size=14741) 2024-11-14T09:50:03,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cd875e2b5ea94968a8c1a45e3cae2d6e 2024-11-14T09:50:03,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/b8d9f24cf36544bb900965693c22b2ee is 50, key is test_row_0/B:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:03,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742348_1524 (size=12301) 2024-11-14T09:50:03,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/b8d9f24cf36544bb900965693c22b2ee 2024-11-14T09:50:03,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ee3b7fe100314170801e2a5dbd8310cd is 50, key is test_row_0/C:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577863900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:03,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742349_1525 (size=12301) 2024-11-14T09:50:03,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T09:50:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:03,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:03,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:04,029 INFO [Thread-1949 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1949. 2024-11-14T09:50:04,029 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-14T09:50:04,029 DEBUG [Thread-1949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,029 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,029 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,029 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,029 DEBUG [Thread-1949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,029 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,029 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:04,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577864025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:04,031 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-14T09:50:04,031 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,031 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,031 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,044 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-14T09:50:04,044 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,044 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,044 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,045 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-14T09:50:04,045 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,045 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,045 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T09:50:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54128 deadline: 1731577864235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 , the old value is region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., hostname=defc576eb6b7,40243,1731577658084, seqNum=2 because the exception is null or not the one we care about 2024-11-14T09:50:04,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T09:50:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:04,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ee3b7fe100314170801e2a5dbd8310cd 2024-11-14T09:50:04,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/cd875e2b5ea94968a8c1a45e3cae2d6e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e 2024-11-14T09:50:04,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e, entries=200, sequenceid=443, filesize=14.4 K 2024-11-14T09:50:04,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/b8d9f24cf36544bb900965693c22b2ee as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee 2024-11-14T09:50:04,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee, entries=150, sequenceid=443, filesize=12.0 K 2024-11-14T09:50:04,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/ee3b7fe100314170801e2a5dbd8310cd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd 2024-11-14T09:50:04,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd, entries=150, sequenceid=443, filesize=12.0 K 2024-11-14T09:50:04,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 76f327b9ee0a90edb9e6efb3d43a25dd in 595ms, sequenceid=443, compaction requested=true 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:04,349 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:04,349 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76f327b9ee0a90edb9e6efb3d43a25dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:04,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:04,350 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:04,350 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:04,350 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/B is initiating minor compaction (all files) 2024-11-14T09:50:04,350 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/A is initiating minor compaction (all files) 2024-11-14T09:50:04,350 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/A in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,350 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/B in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,350 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/0c4ff2712b1a4a8d890d52bd05546930, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=49.0 K 2024-11-14T09:50:04,350 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e920246cb98541bc9124aa01f1dbce35, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=53.7 K 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c4ff2712b1a4a8d890d52bd05546930, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e920246cb98541bc9124aa01f1dbce35, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8ff07e386c464e8e1928aa2392bf61, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731577800753 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5770bc75aa9a495d963e33bf5cc0458f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731577800753 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 17eb0cd42164405ba598b86dd26b25c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731577802030 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 921e2254d2db49989cf79d439d8f3431, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731577802030 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d9f24cf36544bb900965693c22b2ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731577803347 2024-11-14T09:50:04,351 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd875e2b5ea94968a8c1a45e3cae2d6e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731577803347 2024-11-14T09:50:04,360 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#B#compaction#456 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:04,361 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/09940dd4e3f749e78c11d749034822b6 is 50, key is test_row_0/B:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:04,369 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#A#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:04,370 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/159b3c22940e40468d452ff12a10357c is 50, key is test_row_0/A:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:04,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742350_1526 (size=13391) 2024-11-14T09:50:04,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742351_1527 (size=13391) 2024-11-14T09:50:04,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-14T09:50:04,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,395 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T09:50:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/516719152ae247c1bc0415cf6a7e990e is 50, key is test_row_0/A:col10/1731577803863/Put/seqid=0 2024-11-14T09:50:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742352_1528 (size=12301) 2024-11-14T09:50:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:04,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. as already flushing 2024-11-14T09:50:04,546 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-14T09:50:04,546 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,546 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,546 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,716 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-14T09:50:04,716 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:04,716 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,716 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:04,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:04,775 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/09940dd4e3f749e78c11d749034822b6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09940dd4e3f749e78c11d749034822b6 2024-11-14T09:50:04,779 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/B of 76f327b9ee0a90edb9e6efb3d43a25dd into 09940dd4e3f749e78c11d749034822b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:04,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:04,779 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/B, priority=12, startTime=1731577804349; duration=0sec 2024-11-14T09:50:04,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:04,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:B 2024-11-14T09:50:04,779 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:04,780 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:04,780 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): 76f327b9ee0a90edb9e6efb3d43a25dd/C is initiating minor compaction (all files) 2024-11-14T09:50:04,780 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76f327b9ee0a90edb9e6efb3d43a25dd/C in TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:04,781 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7b25f387dbf240af80056bba771b8f24, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp, totalSize=49.0 K 2024-11-14T09:50:04,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b25f387dbf240af80056bba771b8f24, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731577800540 2024-11-14T09:50:04,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 28b2f4d26f414953b5c729ac29c9f4ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731577800753 2024-11-14T09:50:04,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d1d670e0c44d0db279b5345edb099d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1731577802030 2024-11-14T09:50:04,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3b7fe100314170801e2a5dbd8310cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731577803347 2024-11-14T09:50:04,792 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/159b3c22940e40468d452ff12a10357c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/159b3c22940e40468d452ff12a10357c 2024-11-14T09:50:04,795 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76f327b9ee0a90edb9e6efb3d43a25dd#C#compaction#459 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:04,796 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/f409a25dc97a4d888e9a0fa53fe597eb is 50, key is test_row_0/C:col10/1731577803393/Put/seqid=0 2024-11-14T09:50:04,797 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/A of 76f327b9ee0a90edb9e6efb3d43a25dd into 159b3c22940e40468d452ff12a10357c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:04,797 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:04,797 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/A, priority=12, startTime=1731577804349; duration=0sec 2024-11-14T09:50:04,797 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:04,797 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:A 2024-11-14T09:50:04,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742353_1529 (size=13391) 2024-11-14T09:50:04,802 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/516719152ae247c1bc0415cf6a7e990e 2024-11-14T09:50:04,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/52c1ba17293b464faa479f85eb4d5429 is 50, key is test_row_0/B:col10/1731577803863/Put/seqid=0 2024-11-14T09:50:04,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742354_1530 (size=12301) 2024-11-14T09:50:05,204 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/f409a25dc97a4d888e9a0fa53fe597eb as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/f409a25dc97a4d888e9a0fa53fe597eb 2024-11-14T09:50:05,207 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76f327b9ee0a90edb9e6efb3d43a25dd/C of 76f327b9ee0a90edb9e6efb3d43a25dd into f409a25dc97a4d888e9a0fa53fe597eb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:05,207 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:05,207 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., storeName=76f327b9ee0a90edb9e6efb3d43a25dd/C, priority=12, startTime=1731577804349; duration=0sec 2024-11-14T09:50:05,208 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:05,208 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76f327b9ee0a90edb9e6efb3d43a25dd:C 2024-11-14T09:50:05,218 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/52c1ba17293b464faa479f85eb4d5429 2024-11-14T09:50:05,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/59cd1e357f0e47c59b398789ef1f4ad9 is 50, key is test_row_0/C:col10/1731577803863/Put/seqid=0 2024-11-14T09:50:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742355_1531 (size=12301) 2024-11-14T09:50:05,627 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/59cd1e357f0e47c59b398789ef1f4ad9 2024-11-14T09:50:05,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/516719152ae247c1bc0415cf6a7e990e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/516719152ae247c1bc0415cf6a7e990e 2024-11-14T09:50:05,637 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/516719152ae247c1bc0415cf6a7e990e, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T09:50:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/52c1ba17293b464faa479f85eb4d5429 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/52c1ba17293b464faa479f85eb4d5429 2024-11-14T09:50:05,641 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/52c1ba17293b464faa479f85eb4d5429, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T09:50:05,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/59cd1e357f0e47c59b398789ef1f4ad9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/59cd1e357f0e47c59b398789ef1f4ad9 2024-11-14T09:50:05,647 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/59cd1e357f0e47c59b398789ef1f4ad9, entries=150, sequenceid=458, filesize=12.0 K 2024-11-14T09:50:05,648 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=13.42 KB/13740 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1252ms, sequenceid=458, compaction requested=false 2024-11-14T09:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 76f327b9ee0a90edb9e6efb3d43a25dd: 2024-11-14T09:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-14T09:50:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-14T09:50:05,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T09:50:05,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0210 sec 2024-11-14T09:50:05,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0260 sec 2024-11-14T09:50:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T09:50:05,775 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:06,376 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:50:07,466 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-14T09:50:07,466 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:07,466 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,466 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:07,487 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-14T09:50:07,487 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-14T09:50:07,487 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:07,487 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:07,487 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,487 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,487 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,487 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:07,487 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1510 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4529 rows 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4540 rows 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1508 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4524 rows 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1510 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4526 rows 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1502 2024-11-14T09:50:07,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4505 rows 2024-11-14T09:50:07,488 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:50:07,488 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:50:07,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:07,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:07,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:50:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:07,495 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577807495"}]},"ts":"1731577807495"} 2024-11-14T09:50:07,496 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:50:07,496 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:50:07,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:50:07,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, UNASSIGN}] 2024-11-14T09:50:07,499 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, UNASSIGN 2024-11-14T09:50:07,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=76f327b9ee0a90edb9e6efb3d43a25dd, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:07,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, UNASSIGN because future has completed 2024-11-14T09:50:07,502 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:50:07,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:50:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:07,654 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:07,654 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:50:07,655 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 76f327b9ee0a90edb9e6efb3d43a25dd, disabling compactions & flushes 2024-11-14T09:50:07,655 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:07,655 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:07,655 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. after waiting 0 ms 2024-11-14T09:50:07,655 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:07,655 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 76f327b9ee0a90edb9e6efb3d43a25dd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T09:50:07,659 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/1975a8aa332341099f093e23669045ad is 50, key is test_row_0/A:col10/1731577807465/Put/seqid=0 2024-11-14T09:50:07,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742356_1532 (size=12301) 2024-11-14T09:50:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:08,063 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/1975a8aa332341099f093e23669045ad 2024-11-14T09:50:08,069 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/235cd7438814421381f3d847d74cebf8 is 50, key is test_row_0/B:col10/1731577807465/Put/seqid=0 2024-11-14T09:50:08,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742357_1533 (size=12301) 2024-11-14T09:50:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:08,473 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/235cd7438814421381f3d847d74cebf8 2024-11-14T09:50:08,479 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/efe2bd3fea18408d92c37851e74a3b8d is 50, key is test_row_0/C:col10/1731577807465/Put/seqid=0 2024-11-14T09:50:08,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742358_1534 (size=12301) 2024-11-14T09:50:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:08,883 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/efe2bd3fea18408d92c37851e74a3b8d 2024-11-14T09:50:08,886 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/A/1975a8aa332341099f093e23669045ad as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/1975a8aa332341099f093e23669045ad 2024-11-14T09:50:08,889 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/1975a8aa332341099f093e23669045ad, entries=150, sequenceid=469, filesize=12.0 K 2024-11-14T09:50:08,890 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/B/235cd7438814421381f3d847d74cebf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/235cd7438814421381f3d847d74cebf8 2024-11-14T09:50:08,893 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/235cd7438814421381f3d847d74cebf8, entries=150, sequenceid=469, filesize=12.0 K 2024-11-14T09:50:08,893 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/.tmp/C/efe2bd3fea18408d92c37851e74a3b8d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/efe2bd3fea18408d92c37851e74a3b8d 2024-11-14T09:50:08,896 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/efe2bd3fea18408d92c37851e74a3b8d, entries=150, sequenceid=469, filesize=12.0 K 2024-11-14T09:50:08,897 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1242ms, sequenceid=469, compaction requested=true 2024-11-14T09:50:08,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/250e8586f86e41aeb2578b0dde9387e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/061095f6a45343c99f344a39e72e7a5b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/db52b8ffc9204b7e8907cfb07502767f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/490177fc2f734d768bc25ab937d9e944, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1670e3a22844be2ab713e7bbf7adaf5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/23279af0df0e4734ae9749b93a22f448, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/ece00ce9585148d1b5b088462d669c76, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/46cc4ea6b4364c61af647a6c04806cce, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e920246cb98541bc9124aa01f1dbce35, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e] to archive 2024-11-14T09:50:08,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:08,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/065576739fc64c65a4712610b45ebc9e 2024-11-14T09:50:08,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f36585fe43474109b644f9f56029d0bd 2024-11-14T09:50:08,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/250e8586f86e41aeb2578b0dde9387e7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/250e8586f86e41aeb2578b0dde9387e7 2024-11-14T09:50:08,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8c49421285e848ba8b6acbecabbe2c55 2024-11-14T09:50:08,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/790d2eb978964ff1b3e2191f15543a9f 2024-11-14T09:50:08,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/8a2c3f2af0284d44b10e1dc8ec75c185 2024-11-14T09:50:08,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/061095f6a45343c99f344a39e72e7a5b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/061095f6a45343c99f344a39e72e7a5b 2024-11-14T09:50:08,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/c51a32aa48a54f00b50e9c82221be0d7 2024-11-14T09:50:08,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/239b037afda74e86a3cc3b834635d206 2024-11-14T09:50:08,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/db52b8ffc9204b7e8907cfb07502767f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/db52b8ffc9204b7e8907cfb07502767f 2024-11-14T09:50:08,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/85c93aff399f4243bb033b4478972ccf 2024-11-14T09:50:08,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/5124c7c148bd4b7c9c2753849de3ed06 2024-11-14T09:50:08,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/966c7f2ad1ea4bf89f45a4dd248452be 2024-11-14T09:50:08,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/490177fc2f734d768bc25ab937d9e944 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/490177fc2f734d768bc25ab937d9e944 2024-11-14T09:50:08,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/60850a81e8b445adb2ee8c9dc1c96711 2024-11-14T09:50:08,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/b5429074cdf14dadad8431d65f6efac2 2024-11-14T09:50:08,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1670e3a22844be2ab713e7bbf7adaf5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1670e3a22844be2ab713e7bbf7adaf5 2024-11-14T09:50:08,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/761e6cf76a714a108d25c756cb5e0ac2 2024-11-14T09:50:08,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e9c5a5c06e5b4dceba5813275492b3c9 2024-11-14T09:50:08,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/23279af0df0e4734ae9749b93a22f448 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/23279af0df0e4734ae9749b93a22f448 2024-11-14T09:50:08,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/abebd4fc3d2a4d13816549162eecce35 2024-11-14T09:50:08,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/f1995166ab7a47e389fdbbec91f20e4e 2024-11-14T09:50:08,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/ece00ce9585148d1b5b088462d669c76 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/ece00ce9585148d1b5b088462d669c76 2024-11-14T09:50:08,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/987691057f144805a2db3eaba2ea900d 2024-11-14T09:50:08,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cffe52a33db84563af2bf96a77403067 2024-11-14T09:50:08,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/46cc4ea6b4364c61af647a6c04806cce to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/46cc4ea6b4364c61af647a6c04806cce 2024-11-14T09:50:08,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e1fb6ab85b31417db3f646d64f78e15a 2024-11-14T09:50:08,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/932238241bf44f75aa3c3f847db8c4d3 2024-11-14T09:50:08,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e920246cb98541bc9124aa01f1dbce35 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/e920246cb98541bc9124aa01f1dbce35 2024-11-14T09:50:08,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/7f8ff07e386c464e8e1928aa2392bf61 2024-11-14T09:50:08,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/921e2254d2db49989cf79d439d8f3431 2024-11-14T09:50:08,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/cd875e2b5ea94968a8c1a45e3cae2d6e 2024-11-14T09:50:08,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3c3f4fa886514a20a29db435deb82374, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/50a149ef786d4d5cb11a4b42bb77d17e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cffb7f6ac906432ba2807418a527b3c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cd9cbe2aeee2400387ea4e305cb0dc51, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e7bebdfd9e8a424991b85a44e68cc540, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/d11fabea106a454184078defecc539f2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09b02c05c673459ca1b6ebc9cf8ef40c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c9b9221fedd4f7291363915ceb13502, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/0c4ff2712b1a4a8d890d52bd05546930, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee] to archive 2024-11-14T09:50:08,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:08,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3d971fcfce5d4bddb4cec1230349af69 2024-11-14T09:50:08,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/f7acb146ff4d4f228a16a896e6faf2bc 2024-11-14T09:50:08,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3c3f4fa886514a20a29db435deb82374 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/3c3f4fa886514a20a29db435deb82374 2024-11-14T09:50:08,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/be9ab4bf703648c9896fc845f9c27abf 2024-11-14T09:50:08,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/54dfd340c1ee4737838906775379a429 2024-11-14T09:50:08,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/50a149ef786d4d5cb11a4b42bb77d17e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/50a149ef786d4d5cb11a4b42bb77d17e 2024-11-14T09:50:08,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/4d678c956d5b44d9a57ce696fa907c53 2024-11-14T09:50:08,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c3c87386a005403c83a94cfbf250ca7c 2024-11-14T09:50:08,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cffb7f6ac906432ba2807418a527b3c6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cffb7f6ac906432ba2807418a527b3c6 2024-11-14T09:50:08,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/6ad4ee91716a4faabb8fb319446e4ca5 2024-11-14T09:50:08,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/03cd0e1404d34f1980bc31391bdc8a1e 2024-11-14T09:50:08,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/2f7e529f0c1c4bcab3ff10cd2ac6ee8d 2024-11-14T09:50:08,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cd9cbe2aeee2400387ea4e305cb0dc51 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/cd9cbe2aeee2400387ea4e305cb0dc51 2024-11-14T09:50:08,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/a06a17c5829b46b5aed462e561491aa4 2024-11-14T09:50:08,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/1013844f7c844f709f3e8e01a57f9ad0 2024-11-14T09:50:08,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e7bebdfd9e8a424991b85a44e68cc540 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e7bebdfd9e8a424991b85a44e68cc540 2024-11-14T09:50:08,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7f586f7891334db883954c2f3800106b 2024-11-14T09:50:08,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/e78b63ad99204f0fb0d72d8c3a604632 2024-11-14T09:50:08,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/d11fabea106a454184078defecc539f2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/d11fabea106a454184078defecc539f2 2024-11-14T09:50:08,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/35086dfc0e1742d699c719b8f3fc73e7 2024-11-14T09:50:08,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/154c946303ce41959a0c633c4c7797e3 2024-11-14T09:50:08,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09b02c05c673459ca1b6ebc9cf8ef40c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09b02c05c673459ca1b6ebc9cf8ef40c 2024-11-14T09:50:08,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/ed146ac18c07422088cff6b1df1762eb 2024-11-14T09:50:08,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/9dd2d8b293d74332b06dd0e30e752a9b 2024-11-14T09:50:08,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c9b9221fedd4f7291363915ceb13502 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c9b9221fedd4f7291363915ceb13502 2024-11-14T09:50:08,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/7e5a0a24e7a646189389d303d1148e4b 2024-11-14T09:50:08,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/c2341823fab64a739656509eeec96820 2024-11-14T09:50:08,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/0c4ff2712b1a4a8d890d52bd05546930 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/0c4ff2712b1a4a8d890d52bd05546930 2024-11-14T09:50:08,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/8c53dc074687491ba7cdda72337d1f19 2024-11-14T09:50:08,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/5770bc75aa9a495d963e33bf5cc0458f 2024-11-14T09:50:08,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/17eb0cd42164405ba598b86dd26b25c3 2024-11-14T09:50:08,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/b8d9f24cf36544bb900965693c22b2ee 2024-11-14T09:50:08,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/260916778ece4c6e915a846c7ad42906, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/86dbb1e290354c85ab4af032d2fc9aab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6fadcae5242e45f08802b45327dd22d9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0315bd3679ff4abc90533e9bc2be2119, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/d97348b84fd244c6aa4b48cdb9da506c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/76589559678042e6ae5094b8e7e4d23b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8f7c37078f0b43148010df82f510ab6f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/9db1f7b720a04c7f830998ac4264ce59, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7b25f387dbf240af80056bba771b8f24, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd] to archive 2024-11-14T09:50:08,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:08,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/eaeedf25c1364015b208637c4c38b1b6 2024-11-14T09:50:08,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7183e5a8f23344f0a413f8b3048fc91b 2024-11-14T09:50:08,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/260916778ece4c6e915a846c7ad42906 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/260916778ece4c6e915a846c7ad42906 2024-11-14T09:50:08,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/45b558b731ee4f0b9c6c286f444795f1 2024-11-14T09:50:08,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ce48a6865d494e4895c2e28eef3ea782 2024-11-14T09:50:08,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/86dbb1e290354c85ab4af032d2fc9aab to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/86dbb1e290354c85ab4af032d2fc9aab 2024-11-14T09:50:08,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0407591a827746f0954a3b39d3a6b182 2024-11-14T09:50:08,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/a6f9b0306ba84cd0b2ce375f7acf8155 2024-11-14T09:50:08,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6fadcae5242e45f08802b45327dd22d9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6fadcae5242e45f08802b45327dd22d9 2024-11-14T09:50:08,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b9ab2d700fb5436483e44a91bb154b49 2024-11-14T09:50:08,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c8d4c78f8ae741b5b2454cac819afe86 2024-11-14T09:50:08,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/631659ae4c9542de80ed4e9d69027c67 2024-11-14T09:50:08,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0315bd3679ff4abc90533e9bc2be2119 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/0315bd3679ff4abc90533e9bc2be2119 2024-11-14T09:50:08,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/6c909e4cf3a54565a2c4880121ea679f 2024-11-14T09:50:08,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8be6f821289b40edae55f2937b0c5d31 2024-11-14T09:50:08,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/d97348b84fd244c6aa4b48cdb9da506c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/d97348b84fd244c6aa4b48cdb9da506c 2024-11-14T09:50:08,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/5fd43e9e64434465919d70a4b336890f 2024-11-14T09:50:08,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/cf51d9ae631b488ca08806817df02529 2024-11-14T09:50:08,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/76589559678042e6ae5094b8e7e4d23b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/76589559678042e6ae5094b8e7e4d23b 2024-11-14T09:50:08,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c56d1d0a4e71444a99c2e191948e2d0f 2024-11-14T09:50:08,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/61984e7e673b46518dbb132d33a88cf8 2024-11-14T09:50:08,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8f7c37078f0b43148010df82f510ab6f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/8f7c37078f0b43148010df82f510ab6f 2024-11-14T09:50:08,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/93a8f55410a34917a961b635a26cd8c9 2024-11-14T09:50:08,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/b4cdcbc5342649bca58d79e2bf733b44 2024-11-14T09:50:08,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/9db1f7b720a04c7f830998ac4264ce59 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/9db1f7b720a04c7f830998ac4264ce59 2024-11-14T09:50:08,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/c3a6dbe0370f45f2843655309bf1d513 2024-11-14T09:50:08,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/87a96f5a232742e9af6d4e6345ff8a27 2024-11-14T09:50:08,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7b25f387dbf240af80056bba771b8f24 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/7b25f387dbf240af80056bba771b8f24 2024-11-14T09:50:08,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/1370a25627444b7588733247d751495c 2024-11-14T09:50:08,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/28b2f4d26f414953b5c729ac29c9f4ba 2024-11-14T09:50:08,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/64d1d670e0c44d0db279b5345edb099d 2024-11-14T09:50:08,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/ee3b7fe100314170801e2a5dbd8310cd 2024-11-14T09:50:08,980 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-11-14T09:50:08,981 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. 2024-11-14T09:50:08,981 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 76f327b9ee0a90edb9e6efb3d43a25dd: Waiting for close lock at 1731577807654Running coprocessor pre-close hooks at 1731577807654Disabling compacts and flushes for region at 1731577807654Disabling writes for close at 1731577807655 (+1 ms)Obtaining lock to block concurrent updates at 1731577807655Preparing flush snapshotting stores in 76f327b9ee0a90edb9e6efb3d43a25dd at 1731577807655Finished memstore snapshotting TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731577807655Flushing stores of TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd. at 1731577807656 (+1 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/A: creating writer at 1731577807656Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/A: appending metadata at 1731577807658 (+2 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/A: closing flushed file at 1731577807658Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/B: creating writer at 1731577808066 (+408 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/B: appending metadata at 1731577808068 (+2 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/B: closing flushed file at 1731577808068Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/C: creating writer at 1731577808476 (+408 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/C: appending metadata at 1731577808478 (+2 ms)Flushing 76f327b9ee0a90edb9e6efb3d43a25dd/C: closing flushed file at 1731577808478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@647e78b6: reopening flushed file at 1731577808885 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@146cde9c: reopening flushed file at 1731577808889 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72f30ca9: reopening flushed file at 1731577808893 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 76f327b9ee0a90edb9e6efb3d43a25dd in 1242ms, sequenceid=469, compaction requested=true at 1731577808897 (+4 ms)Writing region close event to WAL at 1731577808978 (+81 ms)Running coprocessor post-close hooks at 1731577808981 (+3 ms)Closed at 1731577808981 2024-11-14T09:50:08,983 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:08,983 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=76f327b9ee0a90edb9e6efb3d43a25dd, regionState=CLOSED 2024-11-14T09:50:08,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:50:08,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T09:50:08,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 76f327b9ee0a90edb9e6efb3d43a25dd, server=defc576eb6b7,40243,1731577658084 in 1.4830 sec 2024-11-14T09:50:08,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-14T09:50:08,988 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76f327b9ee0a90edb9e6efb3d43a25dd, UNASSIGN in 1.4890 sec 2024-11-14T09:50:08,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T09:50:08,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4920 sec 2024-11-14T09:50:08,992 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577808992"}]},"ts":"1731577808992"} 2024-11-14T09:50:08,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:50:08,994 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:50:08,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5050 sec 2024-11-14T09:50:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-14T09:50:09,646 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:09,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:50:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,649 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T09:50:09,650 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:09,652 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/recovered.edits] 2024-11-14T09:50:09,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/159b3c22940e40468d452ff12a10357c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/159b3c22940e40468d452ff12a10357c 2024-11-14T09:50:09,655 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/1975a8aa332341099f093e23669045ad to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/1975a8aa332341099f093e23669045ad 2024-11-14T09:50:09,656 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/516719152ae247c1bc0415cf6a7e990e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/A/516719152ae247c1bc0415cf6a7e990e 2024-11-14T09:50:09,657 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09940dd4e3f749e78c11d749034822b6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/09940dd4e3f749e78c11d749034822b6 2024-11-14T09:50:09,658 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/235cd7438814421381f3d847d74cebf8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/235cd7438814421381f3d847d74cebf8 2024-11-14T09:50:09,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/52c1ba17293b464faa479f85eb4d5429 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/B/52c1ba17293b464faa479f85eb4d5429 2024-11-14T09:50:09,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/59cd1e357f0e47c59b398789ef1f4ad9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/59cd1e357f0e47c59b398789ef1f4ad9 2024-11-14T09:50:09,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/efe2bd3fea18408d92c37851e74a3b8d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/efe2bd3fea18408d92c37851e74a3b8d 2024-11-14T09:50:09,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/f409a25dc97a4d888e9a0fa53fe597eb to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/C/f409a25dc97a4d888e9a0fa53fe597eb 2024-11-14T09:50:09,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/recovered.edits/472.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd/recovered.edits/472.seqid 2024-11-14T09:50:09,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/76f327b9ee0a90edb9e6efb3d43a25dd 2024-11-14T09:50:09,665 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:50:09,667 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,669 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:50:09,671 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:50:09,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,672 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:50:09,672 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577809672"}]},"ts":"9223372036854775807"} 2024-11-14T09:50:09,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:50:09,675 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76f327b9ee0a90edb9e6efb3d43a25dd, NAME => 'TestAcidGuarantees,,1731577781795.76f327b9ee0a90edb9e6efb3d43a25dd.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:50:09,675 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:50:09,675 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577809675"}]},"ts":"9223372036854775807"} 2024-11-14T09:50:09,677 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:50:09,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-14T09:50:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-14T09:50:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:50:09,758 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:09,765 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 492) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=4502 (was 4476) - AvailableMemoryMB LEAK? - 2024-11-14T09:50:09,773 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=506, ProcessCount=12, AvailableMemoryMB=4501 2024-11-14T09:50:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:50:09,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:50:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:09,776 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T09:50:09,776 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:09,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-14T09:50:09,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T09:50:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742359_1535 (size=959) 2024-11-14T09:50:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:10,183 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3 2024-11-14T09:50:10,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742360_1536 (size=53) 2024-11-14T09:50:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:10,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:50:10,587 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e6b85e37a52d10a57579fd75d18d566c, disabling compactions & flushes 2024-11-14T09:50:10,587 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. after waiting 0 ms 2024-11-14T09:50:10,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,588 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,588 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6b85e37a52d10a57579fd75d18d566c: Waiting for close lock at 1731577810587Disabling compacts and flushes for region at 1731577810587Disabling writes for close at 1731577810588 (+1 ms)Writing region close event to WAL at 1731577810588Closed at 1731577810588 2024-11-14T09:50:10,589 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T09:50:10,589 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731577810589"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731577810589"}]},"ts":"1731577810589"} 2024-11-14T09:50:10,591 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T09:50:10,593 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T09:50:10,593 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577810593"}]},"ts":"1731577810593"} 2024-11-14T09:50:10,594 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T09:50:10,595 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, ASSIGN}] 2024-11-14T09:50:10,596 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, ASSIGN 2024-11-14T09:50:10,597 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, ASSIGN; state=OFFLINE, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=false 2024-11-14T09:50:10,747 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:10,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, ASSIGN because future has completed 2024-11-14T09:50:10,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:50:10,904 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,904 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:50:10,904 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,904 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:50:10,904 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,904 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,905 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,906 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName A 2024-11-14T09:50:10,906 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:10,907 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:10,907 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,907 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName B 2024-11-14T09:50:10,907 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:10,908 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:10,908 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,909 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName C 2024-11-14T09:50:10,909 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:10,909 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:10,909 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,910 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,910 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,911 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,911 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,911 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:50:10,912 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,913 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T09:50:10,914 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened e6b85e37a52d10a57579fd75d18d566c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59179797, jitterRate=-0.11815230548381805}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:50:10,914 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:10,915 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for e6b85e37a52d10a57579fd75d18d566c: Running coprocessor pre-open hook at 1731577810904Writing region info on filesystem at 1731577810904Initializing all the Stores at 1731577810905 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577810905Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577810905Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577810905Cleaning up temporary data from old regions at 1731577810911 (+6 ms)Running coprocessor post-open hooks at 1731577810914 (+3 ms)Region opened successfully at 1731577810915 (+1 ms) 2024-11-14T09:50:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:10,915 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., pid=144, masterSystemTime=1731577810901 2024-11-14T09:50:10,917 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,917 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:10,917 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=OPEN, openSeqNum=2, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:10,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:50:10,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T09:50:10,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 in 169 msec 2024-11-14T09:50:10,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-14T09:50:10,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, ASSIGN in 325 msec 2024-11-14T09:50:10,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T09:50:10,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577810922"}]},"ts":"1731577810922"} 2024-11-14T09:50:10,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T09:50:10,924 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T09:50:10,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-14T09:50:11,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-14T09:50:11,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:11,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:11,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:11,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:11,928 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:11,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c812665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:11,929 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:11,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:11,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3982de47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:11,931 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:11,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:11,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:11,933 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T09:50:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is defc576eb6b7,35261,1731577657311 2024-11-14T09:50:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36a2e9ea 2024-11-14T09:50:11,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T09:50:11,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T09:50:11,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T09:50:11,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T09:50:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:11,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742361_1537 (size=995) 2024-11-14T09:50:12,347 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T09:50:12,347 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T09:50:12,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:50:12,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE}] 2024-11-14T09:50:12,352 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE 2024-11-14T09:50:12,353 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE because future has completed 2024-11-14T09:50:12,354 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:50:12,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:50:12,507 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,507 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:50:12,507 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing e6b85e37a52d10a57579fd75d18d566c, disabling compactions & flushes 2024-11-14T09:50:12,507 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,507 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,507 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. after waiting 0 ms 2024-11-14T09:50:12,507 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,509 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T09:50:12,510 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,510 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for e6b85e37a52d10a57579fd75d18d566c: Waiting for close lock at 1731577812507Running coprocessor pre-close hooks at 1731577812507Disabling compacts and flushes for region at 1731577812507Disabling writes for close at 1731577812507Writing region close event to WAL at 1731577812507Running coprocessor post-close hooks at 1731577812510 (+3 ms)Closed at 1731577812510 2024-11-14T09:50:12,510 WARN [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: e6b85e37a52d10a57579fd75d18d566c to self. 2024-11-14T09:50:12,512 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,512 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=CLOSED 2024-11-14T09:50:12,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:50:12,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-14T09:50:12,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 in 159 msec 2024-11-14T09:50:12,515 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE; state=CLOSED, location=defc576eb6b7,40243,1731577658084; forceNewPlan=false, retain=true 2024-11-14T09:50:12,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=OPENING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE because future has completed 2024-11-14T09:50:12,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:50:12,822 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,822 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T09:50:12,822 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,822 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T09:50:12,822 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,822 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,823 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,824 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName A 2024-11-14T09:50:12,825 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:12,826 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:12,826 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,826 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName B 2024-11-14T09:50:12,826 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:12,827 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:12,827 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,827 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b85e37a52d10a57579fd75d18d566c columnFamilyName C 2024-11-14T09:50:12,827 DEBUG [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:12,828 INFO [StoreOpener-e6b85e37a52d10a57579fd75d18d566c-1 {}] regionserver.HStore(327): Store=e6b85e37a52d10a57579fd75d18d566c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T09:50:12,828 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,828 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,829 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,830 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,830 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,830 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T09:50:12,831 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,831 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened e6b85e37a52d10a57579fd75d18d566c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75449665, jitterRate=0.12428762018680573}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T09:50:12,832 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,832 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for e6b85e37a52d10a57579fd75d18d566c: Running coprocessor pre-open hook at 1731577812823Writing region info on filesystem at 1731577812823Initializing all the Stores at 1731577812823Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577812823Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577812823Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731577812823Cleaning up temporary data from old regions at 1731577812830 (+7 ms)Running coprocessor post-open hooks at 1731577812832 (+2 ms)Region opened successfully at 1731577812832 2024-11-14T09:50:12,833 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., pid=149, masterSystemTime=1731577812819 2024-11-14T09:50:12,834 DEBUG [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,834 INFO [RS_OPEN_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:12,835 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=OPEN, openSeqNum=5, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:50:12,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-14T09:50:12,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 in 169 msec 2024-11-14T09:50:12,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T09:50:12,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, REOPEN/MOVE in 487 msec 2024-11-14T09:50:12,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T09:50:12,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-14T09:50:12,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-14T09:50:12,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-14T09:50:12,845 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:12,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,847 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbcdfe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5e7f6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,853 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584c9d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@235ba315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e401ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,865 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8a25f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,867 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b786f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e7e09c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d514ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,872 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5079d16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,874 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@101cdd48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,875 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9cbf2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5df76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c236049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,881 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23cbbff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,882 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2531d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679b8dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396c7479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request defc576eb6b7,35261,-1 for getting cluster id 2024-11-14T09:50:12,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T09:50:12,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7792676-cf25-4a20-a379-7c54be1e39c6' 2024-11-14T09:50:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T09:50:12,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7792676-cf25-4a20-a379-7c54be1e39c6" 2024-11-14T09:50:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36834f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [defc576eb6b7,35261,-1] 2024-11-14T09:50:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T09:50:12,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:12,891 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T09:50:12,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20b5eaf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T09:50:12,892 DEBUG [Thread-2409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:12,892 DEBUG [Thread-2392 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T09:50:12,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:12,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,895 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:12,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:12,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,897 DEBUG [Thread-2403 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,897 DEBUG [Thread-2405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,897 DEBUG [Thread-2394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,898 DEBUG [Thread-2398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,898 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,899 DEBUG [Thread-2390 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,899 DEBUG [Thread-2396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,901 DEBUG [Thread-2407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T09:50:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=defc576eb6b7,40243,1731577658084, seqNum=-1] 2024-11-14T09:50:12,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T09:50:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T09:50:12,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5] 2024-11-14T09:50:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:12,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:12,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:12,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577872932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:12,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577872934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577872935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:12,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:12,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146fba755a07714bab99f0fe06e025432d_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577812908/Put/seqid=0 2024-11-14T09:50:12,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577872935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577872936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:12,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742362_1538 (size=12154) 2024-11-14T09:50:13,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577873045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577873045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577873045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577873045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577873045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577873255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577873255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577873256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577873256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577873256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:13,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146fba755a07714bab99f0fe06e025432d_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146fba755a07714bab99f0fe06e025432d_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:13,362 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4e98eecbf1ac4770a6a836fa89ca0433, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:13,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4e98eecbf1ac4770a6a836fa89ca0433 is 175, key is test_row_0/A:col10/1731577812908/Put/seqid=0 2024-11-14T09:50:13,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742363_1539 (size=30955) 2024-11-14T09:50:13,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4e98eecbf1ac4770a6a836fa89ca0433 2024-11-14T09:50:13,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/7e296c7ade8f4dbd80fb211043bef9cf is 50, key is test_row_0/B:col10/1731577812908/Put/seqid=0 2024-11-14T09:50:13,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742364_1540 (size=12001) 2024-11-14T09:50:13,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577873564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577873564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577873565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577873565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,567 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577873565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:13,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/7e296c7ade8f4dbd80fb211043bef9cf 2024-11-14T09:50:13,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:13,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:13,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/86cf6b94a0c948b5a2ce2dcea9a837e7 is 50, key is test_row_0/C:col10/1731577812908/Put/seqid=0 2024-11-14T09:50:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742365_1541 (size=12001) 2024-11-14T09:50:13,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/86cf6b94a0c948b5a2ce2dcea9a837e7 2024-11-14T09:50:13,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4e98eecbf1ac4770a6a836fa89ca0433 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433 2024-11-14T09:50:13,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433, entries=150, sequenceid=15, filesize=30.2 K 2024-11-14T09:50:13,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/7e296c7ade8f4dbd80fb211043bef9cf as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf 2024-11-14T09:50:13,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T09:50:13,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/86cf6b94a0c948b5a2ce2dcea9a837e7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7 2024-11-14T09:50:13,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T09:50:13,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e6b85e37a52d10a57579fd75d18d566c in 944ms, sequenceid=15, compaction requested=false 2024-11-14T09:50:13,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:13,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T09:50:13,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:13,966 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T09:50:13,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144522ac36e50d4744a468b9109ddc50e3_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577812934/Put/seqid=0 2024-11-14T09:50:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742366_1542 (size=12154) 2024-11-14T09:50:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:13,987 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144522ac36e50d4744a468b9109ddc50e3_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144522ac36e50d4744a468b9109ddc50e3_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:13,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/77de1e16fae14992940a66e235c93382, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:13,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/77de1e16fae14992940a66e235c93382 is 175, key is test_row_0/A:col10/1731577812934/Put/seqid=0 2024-11-14T09:50:14,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742367_1543 (size=30955) 2024-11-14T09:50:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:14,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:14,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577874078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577874079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577874079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,083 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577874083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577874083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577874195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577874195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577874195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577874195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577874195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577874405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577874405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577874405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577874405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577874405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,430 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/77de1e16fae14992940a66e235c93382 2024-11-14T09:50:14,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2f4a90a988f4409a9a8738f823721de5 is 50, key is test_row_0/B:col10/1731577812934/Put/seqid=0 2024-11-14T09:50:14,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742368_1544 (size=12001) 2024-11-14T09:50:14,507 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T09:50:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577874715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577874715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577874715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577874715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577874715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:14,846 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2f4a90a988f4409a9a8738f823721de5 2024-11-14T09:50:14,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a4f87fce6087499985a33212dc5f1671 is 50, key is test_row_0/C:col10/1731577812934/Put/seqid=0 2024-11-14T09:50:14,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742369_1545 (size=12001) 2024-11-14T09:50:14,899 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a4f87fce6087499985a33212dc5f1671 2024-11-14T09:50:14,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/77de1e16fae14992940a66e235c93382 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382 2024-11-14T09:50:14,908 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382, entries=150, sequenceid=41, filesize=30.2 K 2024-11-14T09:50:14,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2f4a90a988f4409a9a8738f823721de5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5 2024-11-14T09:50:14,916 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T09:50:14,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a4f87fce6087499985a33212dc5f1671 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671 2024-11-14T09:50:14,922 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T09:50:14,922 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e6b85e37a52d10a57579fd75d18d566c in 956ms, sequenceid=41, compaction requested=false 2024-11-14T09:50:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:14,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T09:50:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T09:50:14,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T09:50:14,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0280 sec 2024-11-14T09:50:14,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0350 sec 2024-11-14T09:50:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T09:50:15,046 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:15,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T09:50:15,048 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:15,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:15,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:15,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T09:50:15,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:15,204 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T09:50:15,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111449505153bb4849249cf413f9eb833085_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577814078/Put/seqid=0 2024-11-14T09:50:15,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742370_1546 (size=12154) 2024-11-14T09:50:15,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:15,222 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111449505153bb4849249cf413f9eb833085_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449505153bb4849249cf413f9eb833085_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/428975ec1c344bb99e3fe4bf896a0fab, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:15,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/428975ec1c344bb99e3fe4bf896a0fab is 175, key is test_row_0/A:col10/1731577814078/Put/seqid=0 2024-11-14T09:50:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:15,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:15,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742371_1547 (size=30955) 2024-11-14T09:50:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577875247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577875247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577875248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577875249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577875251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577875355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577875355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577875355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577875355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577875365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577875565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577875565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577875565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577875565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577875574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,629 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/428975ec1c344bb99e3fe4bf896a0fab 2024-11-14T09:50:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0b728aa1a2cb4e25820ddf9c4c615c6e is 50, key is test_row_0/B:col10/1731577814078/Put/seqid=0 2024-11-14T09:50:15,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742372_1548 (size=12001) 2024-11-14T09:50:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577875875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577875875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577875875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577875875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577875884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,041 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0b728aa1a2cb4e25820ddf9c4c615c6e 2024-11-14T09:50:16,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6e246a13e6ef459d8ed213f2a7c934ad is 50, key is test_row_0/C:col10/1731577814078/Put/seqid=0 2024-11-14T09:50:16,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742373_1549 (size=12001) 2024-11-14T09:50:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577876384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577876385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577876385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577876385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577876395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:16,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:16,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:16,452 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6e246a13e6ef459d8ed213f2a7c934ad 2024-11-14T09:50:16,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/428975ec1c344bb99e3fe4bf896a0fab as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab 2024-11-14T09:50:16,461 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab, entries=150, sequenceid=51, filesize=30.2 K 2024-11-14T09:50:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0b728aa1a2cb4e25820ddf9c4c615c6e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e 2024-11-14T09:50:16,464 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T09:50:16,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6e246a13e6ef459d8ed213f2a7c934ad as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad 2024-11-14T09:50:16,469 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad, entries=150, sequenceid=51, filesize=11.7 K 2024-11-14T09:50:16,476 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e6b85e37a52d10a57579fd75d18d566c in 1272ms, sequenceid=51, compaction requested=true 2024-11-14T09:50:16,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T09:50:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T09:50:16,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T09:50:16,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-11-14T09:50:16,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4350 sec 2024-11-14T09:50:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T09:50:17,195 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:17,195 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T09:50:17,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:50:17,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:17,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:50:17,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T09:50:17,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:17,350 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T09:50:17,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071ff7c1bebc4decbe10f6673589d631_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577815249/Put/seqid=0 2024-11-14T09:50:17,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742374_1550 (size=12154) 2024-11-14T09:50:17,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577877405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577877405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,409 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:17,409 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577877407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577877409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577877409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:50:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577877515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577877524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577877525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577877725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577877735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577877735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:17,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:17,772 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114071ff7c1bebc4decbe10f6673589d631_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071ff7c1bebc4decbe10f6673589d631_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a5f14be76af44b9f9384e38530bf8569, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:17,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a5f14be76af44b9f9384e38530bf8569 is 175, key is test_row_0/A:col10/1731577815249/Put/seqid=0 2024-11-14T09:50:17,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742375_1551 (size=30955) 2024-11-14T09:50:17,783 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a5f14be76af44b9f9384e38530bf8569 2024-11-14T09:50:17,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/93f6f4e5572b4ed8a1e1de0ff46ef851 is 50, key is test_row_0/B:col10/1731577815249/Put/seqid=0 2024-11-14T09:50:17,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742376_1552 (size=12001) 2024-11-14T09:50:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:50:18,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577878038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577878044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577878044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,213 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/93f6f4e5572b4ed8a1e1de0ff46ef851 2024-11-14T09:50:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32eba3e202da40919bacc9207e0fb77c is 50, key is test_row_0/C:col10/1731577815249/Put/seqid=0 2024-11-14T09:50:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742377_1553 (size=12001) 2024-11-14T09:50:18,229 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32eba3e202da40919bacc9207e0fb77c 2024-11-14T09:50:18,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a5f14be76af44b9f9384e38530bf8569 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569 2024-11-14T09:50:18,240 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569, entries=150, sequenceid=77, filesize=30.2 K 2024-11-14T09:50:18,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/93f6f4e5572b4ed8a1e1de0ff46ef851 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851 2024-11-14T09:50:18,246 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T09:50:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32eba3e202da40919bacc9207e0fb77c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c 2024-11-14T09:50:18,251 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T09:50:18,252 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e6b85e37a52d10a57579fd75d18d566c in 902ms, sequenceid=77, compaction requested=true 2024-11-14T09:50:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:18,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T09:50:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T09:50:18,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T09:50:18,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0560 sec 2024-11-14T09:50:18,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.0620 sec 2024-11-14T09:50:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T09:50:18,334 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:18,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T09:50:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:50:18,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:18,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:18,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:50:18,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T09:50:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:18,490 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T09:50:18,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fab25efab2864406b1d194bde2136837_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577817402/Put/seqid=0 2024-11-14T09:50:18,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742378_1554 (size=12154) 2024-11-14T09:50:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:18,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577878600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577878605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577878606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:50:18,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577878715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577878715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577878715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:18,907 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fab25efab2864406b1d194bde2136837_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fab25efab2864406b1d194bde2136837_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/e1bcf7d1199e448a92b9d2ce1039d87c, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:18,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/e1bcf7d1199e448a92b9d2ce1039d87c is 175, key is test_row_0/A:col10/1731577817402/Put/seqid=0 2024-11-14T09:50:18,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742379_1555 (size=30955) 2024-11-14T09:50:18,912 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/e1bcf7d1199e448a92b9d2ce1039d87c 2024-11-14T09:50:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/44570101b7b5488c95eb2da8f890b6a5 is 50, key is test_row_0/B:col10/1731577817402/Put/seqid=0 2024-11-14T09:50:18,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742380_1556 (size=12001) 2024-11-14T09:50:18,921 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/44570101b7b5488c95eb2da8f890b6a5 2024-11-14T09:50:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/41497f95a272423c995ba34ef6f1cf66 is 50, key is test_row_0/C:col10/1731577817402/Put/seqid=0 2024-11-14T09:50:18,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577878925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577878925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577878926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742381_1557 (size=12001) 2024-11-14T09:50:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:50:19,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577879235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577879235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577879235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,333 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/41497f95a272423c995ba34ef6f1cf66 2024-11-14T09:50:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/e1bcf7d1199e448a92b9d2ce1039d87c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c 2024-11-14T09:50:19,352 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c, entries=150, sequenceid=87, filesize=30.2 K 2024-11-14T09:50:19,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-14T09:50:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/44570101b7b5488c95eb2da8f890b6a5 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5 2024-11-14T09:50:19,363 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5, entries=150, sequenceid=87, filesize=11.7 K 2024-11-14T09:50:19,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/41497f95a272423c995ba34ef6f1cf66 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66 2024-11-14T09:50:19,374 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66, entries=150, sequenceid=87, filesize=11.7 K 2024-11-14T09:50:19,375 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e6b85e37a52d10a57579fd75d18d566c in 885ms, sequenceid=87, compaction requested=true 2024-11-14T09:50:19,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:19,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T09:50:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T09:50:19,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T09:50:19,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-11-14T09:50:19,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0440 sec 2024-11-14T09:50:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:19,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:19,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T09:50:19,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140e3f93f51359414ca50ca3c193afbeaa_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:19,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742382_1558 (size=12154) 2024-11-14T09:50:19,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577879463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577879464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T09:50:19,475 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:19,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T09:50:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:19,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:19,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:19,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:19,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577879574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577879575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:19,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577879744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577879745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577879745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577879785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577879785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:19,861 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:19,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140e3f93f51359414ca50ca3c193afbeaa_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e3f93f51359414ca50ca3c193afbeaa_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:19,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4b09cc55472e49f28cf92a7c8b673141, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:19,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4b09cc55472e49f28cf92a7c8b673141 is 175, key is test_row_0/A:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742383_1559 (size=30955) 2024-11-14T09:50:19,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4b09cc55472e49f28cf92a7c8b673141 2024-11-14T09:50:19,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c175133ad1954163a9622465c489cb99 is 50, key is test_row_0/B:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:19,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742384_1560 (size=12001) 2024-11-14T09:50:19,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:19,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577880095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577880095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:20,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c175133ad1954163a9622465c489cb99 2024-11-14T09:50:20,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/8f150bc56a794de489319bad944617e3 is 50, key is test_row_0/C:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742385_1561 (size=12001) 2024-11-14T09:50:20,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577880608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577880608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:20,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/8f150bc56a794de489319bad944617e3 2024-11-14T09:50:20,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/4b09cc55472e49f28cf92a7c8b673141 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141 2024-11-14T09:50:20,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141, entries=150, sequenceid=114, filesize=30.2 K 2024-11-14T09:50:20,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c175133ad1954163a9622465c489cb99 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99 2024-11-14T09:50:20,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:50:20,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/8f150bc56a794de489319bad944617e3 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3 2024-11-14T09:50:20,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T09:50:20,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e6b85e37a52d10a57579fd75d18d566c in 1297ms, sequenceid=114, compaction requested=true 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:20,724 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:20,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T09:50:20,724 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:50:20,729 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:50:20,729 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:20,729 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 185730 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:50:20,729 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:20,730 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,730 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=181.4 K 2024-11-14T09:50:20,730 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,730 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,730 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=70.3 K 2024-11-14T09:50:20,730 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141] 2024-11-14T09:50:20,731 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e98eecbf1ac4770a6a836fa89ca0433, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731577812905 2024-11-14T09:50:20,731 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e296c7ade8f4dbd80fb211043bef9cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731577812905 2024-11-14T09:50:20,731 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77de1e16fae14992940a66e235c93382, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577812919 2024-11-14T09:50:20,731 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4a90a988f4409a9a8738f823721de5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577812919 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428975ec1c344bb99e3fe4bf896a0fab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731577814078 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b728aa1a2cb4e25820ddf9c4c615c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731577814078 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f14be76af44b9f9384e38530bf8569, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577815246 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f6f4e5572b4ed8a1e1de0ff46ef851, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577815246 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1bcf7d1199e448a92b9d2ce1039d87c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731577817402 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 44570101b7b5488c95eb2da8f890b6a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731577817402 2024-11-14T09:50:20,732 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b09cc55472e49f28cf92a7c8b673141, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:20,733 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c175133ad1954163a9622465c489cb99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:20,739 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:20,741 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114fdc0da45f9e44e628e636badadf2fcf4_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:20,742 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#484 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:20,742 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/db680d9f6b584b458bb79d7a9b888c02 is 50, key is test_row_0/B:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:20,744 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114fdc0da45f9e44e628e636badadf2fcf4_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:20,745 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fdc0da45f9e44e628e636badadf2fcf4_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:20,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742386_1562 (size=4469) 2024-11-14T09:50:20,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742387_1563 (size=12207) 2024-11-14T09:50:20,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:20,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:20,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:20,765 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/db680d9f6b584b458bb79d7a9b888c02 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/db680d9f6b584b458bb79d7a9b888c02 2024-11-14T09:50:20,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f8ec27edc8884f3abbefa60245610c7c_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577819463/Put/seqid=0 2024-11-14T09:50:20,770 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into db680d9f6b584b458bb79d7a9b888c02(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:20,770 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:20,770 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=10, startTime=1731577820724; duration=0sec 2024-11-14T09:50:20,770 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:20,770 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:20,770 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T09:50:20,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742388_1564 (size=14594) 2024-11-14T09:50:20,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:20,776 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T09:50:20,776 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:20,776 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,776 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=70.3 K 2024-11-14T09:50:20,777 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 86cf6b94a0c948b5a2ce2dcea9a837e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731577812905 2024-11-14T09:50:20,777 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f87fce6087499985a33212dc5f1671, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731577812919 2024-11-14T09:50:20,780 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e246a13e6ef459d8ed213f2a7c934ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731577814078 2024-11-14T09:50:20,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 32eba3e202da40919bacc9207e0fb77c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731577815246 2024-11-14T09:50:20,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 41497f95a272423c995ba34ef6f1cf66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731577817402 2024-11-14T09:50:20,781 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f150bc56a794de489319bad944617e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:20,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f8ec27edc8884f3abbefa60245610c7c_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8ec27edc8884f3abbefa60245610c7c_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:20,785 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/150ace4093d349f38eb459c06d4dea27, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:20,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/150ace4093d349f38eb459c06d4dea27 is 175, key is test_row_0/A:col10/1731577819463/Put/seqid=0 2024-11-14T09:50:20,794 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:20,794 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/b45d6451768b4b218c15dece019bc4a4 is 50, key is test_row_0/C:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:20,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742389_1565 (size=39549) 2024-11-14T09:50:20,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=125, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/150ace4093d349f38eb459c06d4dea27 2024-11-14T09:50:20,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742390_1566 (size=12207) 2024-11-14T09:50:20,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c38564a4c1a8458e8c841398ccb9610d is 50, key is test_row_0/B:col10/1731577819463/Put/seqid=0 2024-11-14T09:50:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742391_1567 (size=12001) 2024-11-14T09:50:20,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577880815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577880818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,822 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577880821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:20,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577880925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577880925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577880934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577881134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577881134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,139 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577881145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,159 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:21,159 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/2367232e7b0f49d187faafa88f14342c is 175, key is test_row_0/A:col10/1731577819427/Put/seqid=0 2024-11-14T09:50:21,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:21,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742392_1568 (size=31161) 2024-11-14T09:50:21,169 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/2367232e7b0f49d187faafa88f14342c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c 2024-11-14T09:50:21,175 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into 2367232e7b0f49d187faafa88f14342c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:21,175 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:21,175 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=10, startTime=1731577820724; duration=0sec 2024-11-14T09:50:21,175 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:21,175 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:21,209 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/b45d6451768b4b218c15dece019bc4a4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/b45d6451768b4b218c15dece019bc4a4 2024-11-14T09:50:21,214 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into b45d6451768b4b218c15dece019bc4a4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:21,214 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:21,214 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=10, startTime=1731577820724; duration=0sec 2024-11-14T09:50:21,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c38564a4c1a8458e8c841398ccb9610d 2024-11-14T09:50:21,214 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:21,214 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:21,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/229a1c9e1f434b6bbe96d7d3552de014 is 50, key is test_row_0/C:col10/1731577819463/Put/seqid=0 2024-11-14T09:50:21,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742393_1569 (size=12001) 2024-11-14T09:50:21,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/229a1c9e1f434b6bbe96d7d3552de014 2024-11-14T09:50:21,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/150ace4093d349f38eb459c06d4dea27 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27 2024-11-14T09:50:21,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27, entries=200, sequenceid=125, filesize=38.6 K 2024-11-14T09:50:21,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/c38564a4c1a8458e8c841398ccb9610d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d 2024-11-14T09:50:21,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d, entries=150, sequenceid=125, filesize=11.7 K 2024-11-14T09:50:21,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/229a1c9e1f434b6bbe96d7d3552de014 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014 2024-11-14T09:50:21,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014, entries=150, sequenceid=125, filesize=11.7 K 2024-11-14T09:50:21,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e6b85e37a52d10a57579fd75d18d566c in 489ms, sequenceid=125, compaction requested=false 2024-11-14T09:50:21,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:21,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T09:50:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,314 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:50:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ab82e30f552841de8d8d9ff96d26a583_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577820819/Put/seqid=0 2024-11-14T09:50:21,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742394_1570 (size=12304) 2024-11-14T09:50:21,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:21,327 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ab82e30f552841de8d8d9ff96d26a583_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab82e30f552841de8d8d9ff96d26a583_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/398e64ee2eee480ba1b58d681a7b7c4b, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/398e64ee2eee480ba1b58d681a7b7c4b is 175, key is test_row_0/A:col10/1731577820819/Put/seqid=0 2024-11-14T09:50:21,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742395_1571 (size=31105) 2024-11-14T09:50:21,332 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/398e64ee2eee480ba1b58d681a7b7c4b 2024-11-14T09:50:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/50ef2b3055124665b68b28ec23b9ce0f is 50, key is test_row_0/B:col10/1731577820819/Put/seqid=0 2024-11-14T09:50:21,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742396_1572 (size=12151) 2024-11-14T09:50:21,344 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/50ef2b3055124665b68b28ec23b9ce0f 2024-11-14T09:50:21,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/4874c46652eb4982ac1cdca5fbd5b381 is 50, key is test_row_0/C:col10/1731577820819/Put/seqid=0 2024-11-14T09:50:21,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742397_1573 (size=12151) 2024-11-14T09:50:21,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577881459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577881462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577881462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577881574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577881575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577881575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577881625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577881625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,627 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,628 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,754 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/4874c46652eb4982ac1cdca5fbd5b381 2024-11-14T09:50:21,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/398e64ee2eee480ba1b58d681a7b7c4b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b 2024-11-14T09:50:21,761 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b, entries=150, sequenceid=153, filesize=30.4 K 2024-11-14T09:50:21,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/50ef2b3055124665b68b28ec23b9ce0f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f 2024-11-14T09:50:21,764 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T09:50:21,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/4874c46652eb4982ac1cdca5fbd5b381 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381 2024-11-14T09:50:21,768 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381, entries=150, sequenceid=153, filesize=11.9 K 2024-11-14T09:50:21,769 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e6b85e37a52d10a57579fd75d18d566c in 455ms, sequenceid=153, compaction requested=true 2024-11-14T09:50:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:21,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T09:50:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T09:50:21,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T09:50:21,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-11-14T09:50:21,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.2970 sec 2024-11-14T09:50:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:21,788 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:21,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T09:50:21,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147d0b023491b1463ca61267419cd530ff_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:21,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742398_1574 (size=12304) 2024-11-14T09:50:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577881826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577881826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577881827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577881934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577881935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:21,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577882145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577882145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577882145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:22,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147d0b023491b1463ca61267419cd530ff_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d0b023491b1463ca61267419cd530ff_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:22,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/731777b830e24da0803050c3ffe34e40, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:22,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/731777b830e24da0803050c3ffe34e40 is 175, key is test_row_0/A:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:22,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742399_1575 (size=31105) 2024-11-14T09:50:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577882455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577882455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577882455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/731777b830e24da0803050c3ffe34e40 2024-11-14T09:50:22,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/086b4929189843d5a6d3acc633c789c9 is 50, key is test_row_0/B:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742400_1576 (size=12151) 2024-11-14T09:50:22,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577882965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577882965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:22,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577882965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:22,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/086b4929189843d5a6d3acc633c789c9 2024-11-14T09:50:23,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/01b07ba3a077435aa8cbe3337ad6e86c is 50, key is test_row_0/C:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:23,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742401_1577 (size=12151) 2024-11-14T09:50:23,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/01b07ba3a077435aa8cbe3337ad6e86c 2024-11-14T09:50:23,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/731777b830e24da0803050c3ffe34e40 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40 2024-11-14T09:50:23,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40, entries=150, sequenceid=168, filesize=30.4 K 2024-11-14T09:50:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/086b4929189843d5a6d3acc633c789c9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9 2024-11-14T09:50:23,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T09:50:23,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/01b07ba3a077435aa8cbe3337ad6e86c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c 2024-11-14T09:50:23,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T09:50:23,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for e6b85e37a52d10a57579fd75d18d566c in 1265ms, sequenceid=168, compaction requested=true 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:23,053 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:23,053 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:23,055 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,055 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,055 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/db680d9f6b584b458bb79d7a9b888c02, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=47.4 K 2024-11-14T09:50:23,055 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=129.8 K 2024-11-14T09:50:23,055 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40] 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting db680d9f6b584b458bb79d7a9b888c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:23,055 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2367232e7b0f49d187faafa88f14342c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c38564a4c1a8458e8c841398ccb9610d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731577819463 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 150ace4093d349f38eb459c06d4dea27, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731577819457 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ef2b3055124665b68b28ec23b9ce0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731577820814 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 398e64ee2eee480ba1b58d681a7b7c4b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731577820814 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 086b4929189843d5a6d3acc633c789c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:23,056 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 731777b830e24da0803050c3ffe34e40, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:23,067 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:23,069 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:23,069 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/f12975dedb6443a1b24545b02c0abb44 is 50, key is test_row_0/B:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:23,073 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114419abc5cd4dc4520b1f3daba6dcafb62_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:23,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114419abc5cd4dc4520b1f3daba6dcafb62_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:23,075 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114419abc5cd4dc4520b1f3daba6dcafb62_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:23,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742402_1578 (size=12493) 2024-11-14T09:50:23,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742403_1579 (size=4469) 2024-11-14T09:50:23,497 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/f12975dedb6443a1b24545b02c0abb44 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f12975dedb6443a1b24545b02c0abb44 2024-11-14T09:50:23,498 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:23,499 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f98d5ff4f0524d57a3b8cc5eb8601736 is 175, key is test_row_0/A:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:23,500 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into f12975dedb6443a1b24545b02c0abb44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:23,500 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:23,500 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=12, startTime=1731577823053; duration=0sec 2024-11-14T09:50:23,500 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:23,501 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:23,501 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:23,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:23,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:23,502 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,502 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/b45d6451768b4b218c15dece019bc4a4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=47.4 K 2024-11-14T09:50:23,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting b45d6451768b4b218c15dece019bc4a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731577818576 2024-11-14T09:50:23,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 229a1c9e1f434b6bbe96d7d3552de014, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731577819463 2024-11-14T09:50:23,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 4874c46652eb4982ac1cdca5fbd5b381, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731577820814 2024-11-14T09:50:23,503 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b07ba3a077435aa8cbe3337ad6e86c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:23,510 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742404_1580 (size=31447) 2024-11-14T09:50:23,511 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a92d10fac32d4668b5e4f607a3a0f636 is 50, key is test_row_0/C:col10/1731577821787/Put/seqid=0 2024-11-14T09:50:23,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742405_1581 (size=12493) 2024-11-14T09:50:23,515 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f98d5ff4f0524d57a3b8cc5eb8601736 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736 2024-11-14T09:50:23,517 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a92d10fac32d4668b5e4f607a3a0f636 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a92d10fac32d4668b5e4f607a3a0f636 2024-11-14T09:50:23,519 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into f98d5ff4f0524d57a3b8cc5eb8601736(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:23,519 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:23,519 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=12, startTime=1731577823053; duration=0sec 2024-11-14T09:50:23,519 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:23,519 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:23,522 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into a92d10fac32d4668b5e4f607a3a0f636(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:23,522 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:23,522 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=12, startTime=1731577823053; duration=0sec 2024-11-14T09:50:23,523 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:23,523 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:23,637 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:23,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:50:23,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114182e161025f1414289bee86180283697_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577821811/Put/seqid=0 2024-11-14T09:50:23,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742406_1582 (size=12304) 2024-11-14T09:50:23,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T09:50:23,655 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:23,655 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T09:50:23,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:23,657 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:23,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:23,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577883662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577883664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577883774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577883775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:23,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:23,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577883975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577883975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577883975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,976 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:23,976 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,976 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577883984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577883984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:23,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:24,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:24,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114182e161025f1414289bee86180283697_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114182e161025f1414289bee86180283697_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:24,053 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/3669283c7be5455a95f1fc22c8722721, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:24,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/3669283c7be5455a95f1fc22c8722721 is 175, key is test_row_0/A:col10/1731577821811/Put/seqid=0 2024-11-14T09:50:24,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742407_1583 (size=31105) 2024-11-14T09:50:24,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577884295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577884295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:24,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:24,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,458 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/3669283c7be5455a95f1fc22c8722721 2024-11-14T09:50:24,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/df9920ac0067494b8463640537e35822 is 50, key is test_row_0/B:col10/1731577821811/Put/seqid=0 2024-11-14T09:50:24,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742408_1584 (size=12151) 2024-11-14T09:50:24,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/df9920ac0067494b8463640537e35822 2024-11-14T09:50:24,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2320854061ac4528b4c929857b8e8160 is 50, key is test_row_0/C:col10/1731577821811/Put/seqid=0 2024-11-14T09:50:24,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742409_1585 (size=12151) 2024-11-14T09:50:24,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:24,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577884805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577884805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:24,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:24,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:24,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:24,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:24,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2320854061ac4528b4c929857b8e8160 2024-11-14T09:50:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/3669283c7be5455a95f1fc22c8722721 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721 2024-11-14T09:50:24,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721, entries=150, sequenceid=195, filesize=30.4 K 2024-11-14T09:50:24,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/df9920ac0067494b8463640537e35822 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822 2024-11-14T09:50:24,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T09:50:24,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2320854061ac4528b4c929857b8e8160 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160 2024-11-14T09:50:24,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T09:50:24,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e6b85e37a52d10a57579fd75d18d566c in 1270ms, sequenceid=195, compaction requested=false 2024-11-14T09:50:24,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:25,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T09:50:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:25,030 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T09:50:25,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469ce85f2388f417cb9a15ed268e79f74_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577823656/Put/seqid=0 2024-11-14T09:50:25,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742410_1586 (size=12304) 2024-11-14T09:50:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,449 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111469ce85f2388f417cb9a15ed268e79f74_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469ce85f2388f417cb9a15ed268e79f74_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/633b5d438ef344d09dbb8ce198ca058c, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/633b5d438ef344d09dbb8ce198ca058c is 175, key is test_row_0/A:col10/1731577823656/Put/seqid=0 2024-11-14T09:50:25,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742411_1587 (size=31105) 2024-11-14T09:50:25,455 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/633b5d438ef344d09dbb8ce198ca058c 2024-11-14T09:50:25,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/9b6eb375d2974c438a596af140bd9176 is 50, key is test_row_0/B:col10/1731577823656/Put/seqid=0 2024-11-14T09:50:25,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742412_1588 (size=12151) 2024-11-14T09:50:25,497 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/9b6eb375d2974c438a596af140bd9176 2024-11-14T09:50:25,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/fbe9577bcc014784b78d3890cc89af5e is 50, key is test_row_0/C:col10/1731577823656/Put/seqid=0 2024-11-14T09:50:25,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742413_1589 (size=12151) 2024-11-14T09:50:25,524 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/fbe9577bcc014784b78d3890cc89af5e 2024-11-14T09:50:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/633b5d438ef344d09dbb8ce198ca058c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c 2024-11-14T09:50:25,533 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c, entries=150, sequenceid=207, filesize=30.4 K 2024-11-14T09:50:25,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/9b6eb375d2974c438a596af140bd9176 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176 2024-11-14T09:50:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,537 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T09:50:25,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/fbe9577bcc014784b78d3890cc89af5e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e 2024-11-14T09:50:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,540 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e, entries=150, sequenceid=207, filesize=11.9 K 2024-11-14T09:50:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,541 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for e6b85e37a52d10a57579fd75d18d566c in 511ms, sequenceid=207, compaction requested=true 2024-11-14T09:50:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:25,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T09:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T09:50:25,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8920 sec 2024-11-14T09:50:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T09:50:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,805 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,805 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T09:50:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,807 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,808 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:25,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:25,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114511281341a6b4f5e82b29b01c8d32f95_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:25,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742414_1590 (size=22268) 2024-11-14T09:50:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:25,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577885947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:25,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577885948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:25,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:25,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:25,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577885985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:25,988 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577885995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:25,999 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577886005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,009 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577886055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577886055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:26,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577886264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577886264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,309 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:26,313 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114511281341a6b4f5e82b29b01c8d32f95_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114511281341a6b4f5e82b29b01c8d32f95_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:26,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/ecdeb18d372240c1b24985ba38c3ece7, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:26,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/ecdeb18d372240c1b24985ba38c3ece7 is 175, key is test_row_0/A:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:26,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742415_1591 (size=65673) 2024-11-14T09:50:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:26,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577886574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577886575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:26,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/ecdeb18d372240c1b24985ba38c3ece7 2024-11-14T09:50:26,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/750b760ed4814229aac63f30dbb488da is 50, key is test_row_0/B:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:26,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742416_1592 (size=12151) 2024-11-14T09:50:26,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:26,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:27,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:27,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:27,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:27,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:27,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577887085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577887085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:27,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:27,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/750b760ed4814229aac63f30dbb488da 2024-11-14T09:50:27,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/cef638abc2d441aa8818957acf5578a8 is 50, key is test_row_0/C:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742417_1593 (size=12151) 2024-11-14T09:50:27,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/cef638abc2d441aa8818957acf5578a8 2024-11-14T09:50:27,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/ecdeb18d372240c1b24985ba38c3ece7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7 2024-11-14T09:50:27,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7, entries=350, sequenceid=218, filesize=64.1 K 2024-11-14T09:50:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/750b760ed4814229aac63f30dbb488da as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da 2024-11-14T09:50:27,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T09:50:27,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/cef638abc2d441aa8818957acf5578a8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8 2024-11-14T09:50:27,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T09:50:27,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e6b85e37a52d10a57579fd75d18d566c in 1294ms, sequenceid=218, compaction requested=true 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:27,177 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:27,177 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:27,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:27,178 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159330 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:27,178 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:27,178 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:27,178 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:27,178 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,178 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,178 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f12975dedb6443a1b24545b02c0abb44, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=47.8 K 2024-11-14T09:50:27,178 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=155.6 K 2024-11-14T09:50:27,178 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,178 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7] 2024-11-14T09:50:27,179 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f12975dedb6443a1b24545b02c0abb44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:27,179 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f98d5ff4f0524d57a3b8cc5eb8601736, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:27,179 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3669283c7be5455a95f1fc22c8722721, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577821811 2024-11-14T09:50:27,179 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633b5d438ef344d09dbb8ce198ca058c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731577823653 2024-11-14T09:50:27,180 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting df9920ac0067494b8463640537e35822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577821811 2024-11-14T09:50:27,180 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecdeb18d372240c1b24985ba38c3ece7, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:27,180 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6eb375d2974c438a596af140bd9176, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731577823653 2024-11-14T09:50:27,180 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 750b760ed4814229aac63f30dbb488da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:27,187 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:27,190 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:27,190 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/5f95f44af18143488d0e9344397583aa is 50, key is test_row_0/B:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:27,193 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111488f08737c3e54901837534f433108ddf_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:27,195 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111488f08737c3e54901837534f433108ddf_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:27,195 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111488f08737c3e54901837534f433108ddf_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:27,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742419_1595 (size=4469) 2024-11-14T09:50:27,203 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#507 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:27,204 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/b5de5ca811bd4359b5691061118e3843 is 175, key is test_row_0/A:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742420_1596 (size=31583) 2024-11-14T09:50:27,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742418_1594 (size=12629) 2024-11-14T09:50:27,211 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/b5de5ca811bd4359b5691061118e3843 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843 2024-11-14T09:50:27,215 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into b5de5ca811bd4359b5691061118e3843(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:27,215 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:27,215 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=12, startTime=1731577827177; duration=0sec 2024-11-14T09:50:27,215 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:27,215 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:27,215 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:27,217 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:27,217 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:27,217 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,217 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a92d10fac32d4668b5e4f607a3a0f636, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=47.8 K 2024-11-14T09:50:27,217 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92d10fac32d4668b5e4f607a3a0f636, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731577821457 2024-11-14T09:50:27,218 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2320854061ac4528b4c929857b8e8160, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731577821811 2024-11-14T09:50:27,218 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe9577bcc014784b78d3890cc89af5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731577823653 2024-11-14T09:50:27,218 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef638abc2d441aa8818957acf5578a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:27,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T09:50:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,222 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T09:50:27,227 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:27,227 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/d24dfd81a7c5448280ec26e53dd11c9f is 50, key is test_row_0/C:col10/1731577825875/Put/seqid=0 2024-11-14T09:50:27,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142f189709dca543fe9bc28d9bfd4be45c_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577825946/Put/seqid=0 2024-11-14T09:50:27,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742422_1598 (size=12304) 2024-11-14T09:50:27,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,249 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142f189709dca543fe9bc28d9bfd4be45c_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142f189709dca543fe9bc28d9bfd4be45c_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/27b17721b6794ef3baf68c96836432e8, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:27,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/27b17721b6794ef3baf68c96836432e8 is 175, key is test_row_0/A:col10/1731577825946/Put/seqid=0 2024-11-14T09:50:27,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742421_1597 (size=12629) 2024-11-14T09:50:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742423_1599 (size=31105) 2024-11-14T09:50:27,269 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/27b17721b6794ef3baf68c96836432e8 2024-11-14T09:50:27,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/5242cb04e37341b5bae0df9d2cca144f is 50, key is test_row_0/B:col10/1731577825946/Put/seqid=0 2024-11-14T09:50:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742424_1600 (size=12151) 2024-11-14T09:50:27,279 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/5242cb04e37341b5bae0df9d2cca144f 2024-11-14T09:50:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/462455b5716147c78d85d9994dedb828 is 50, key is test_row_0/C:col10/1731577825946/Put/seqid=0 2024-11-14T09:50:27,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742425_1601 (size=12151) 2024-11-14T09:50:27,298 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/462455b5716147c78d85d9994dedb828 2024-11-14T09:50:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/27b17721b6794ef3baf68c96836432e8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8 2024-11-14T09:50:27,305 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8, entries=150, sequenceid=244, filesize=30.4 K 2024-11-14T09:50:27,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/5242cb04e37341b5bae0df9d2cca144f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f 2024-11-14T09:50:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,309 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T09:50:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/462455b5716147c78d85d9994dedb828 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828, entries=150, sequenceid=244, filesize=11.9 K 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for e6b85e37a52d10a57579fd75d18d566c in 92ms, sequenceid=244, compaction requested=false 2024-11-14T09:50:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T09:50:27,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-14T09:50:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5120 sec 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,615 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/5f95f44af18143488d0e9344397583aa as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5f95f44af18143488d0e9344397583aa 2024-11-14T09:50:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,619 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into 5f95f44af18143488d0e9344397583aa(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:27,619 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:27,619 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=12, startTime=1731577827177; duration=0sec 2024-11-14T09:50:27,619 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,619 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,661 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/d24dfd81a7c5448280ec26e53dd11c9f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/d24dfd81a7c5448280ec26e53dd11c9f 2024-11-14T09:50:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,666 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into d24dfd81a7c5448280ec26e53dd11c9f(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:27,666 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:27,666 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=12, startTime=1731577827177; duration=0sec 2024-11-14T09:50:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,666 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:27,666 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T09:50:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,955 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,955 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-14T09:50:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,957 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,958 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T09:50:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-14T09:50:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-14T09:50:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T09:50:28,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-14T09:50:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 160 msec 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,164 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:28,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:28,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146c4619dba7f345d8bccfa146f2e2f10e_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742426_1602 (size=14894) 2024-11-14T09:50:28,226 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146c4619dba7f345d8bccfa146f2e2f10e_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146c4619dba7f345d8bccfa146f2e2f10e_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:28,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/85d2494324b444299521b0de26eb09d4, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/85d2494324b444299521b0de26eb09d4 is 175, key is test_row_0/A:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742427_1603 (size=39849) 2024-11-14T09:50:28,238 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/85d2494324b444299521b0de26eb09d4 2024-11-14T09:50:28,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0de50e11e71646169ca2061f73085886 is 50, key is test_row_0/B:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577888244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577888246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742428_1604 (size=12251) 2024-11-14T09:50:28,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0de50e11e71646169ca2061f73085886 2024-11-14T09:50:28,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32da59376e7346d9beadf7277b856c4d is 50, key is test_row_0/C:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T09:50:28,275 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:28,275 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-14T09:50:28,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:28,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:28,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742429_1605 (size=12251) 2024-11-14T09:50:28,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32da59376e7346d9beadf7277b856c4d 2024-11-14T09:50:28,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/85d2494324b444299521b0de26eb09d4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4 2024-11-14T09:50:28,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4, entries=200, sequenceid=258, filesize=38.9 K 2024-11-14T09:50:28,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/0de50e11e71646169ca2061f73085886 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886 2024-11-14T09:50:28,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T09:50:28,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/32da59376e7346d9beadf7277b856c4d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d 2024-11-14T09:50:28,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d, entries=150, sequenceid=258, filesize=12.0 K 2024-11-14T09:50:28,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e6b85e37a52d10a57579fd75d18d566c in 167ms, sequenceid=258, compaction requested=true 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:28,331 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:28,331 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:28,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:28,333 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:28,333 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:28,333 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,333 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5f95f44af18143488d0e9344397583aa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=36.2 K 2024-11-14T09:50:28,333 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:28,333 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:28,333 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,333 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=100.1 K 2024-11-14T09:50:28,333 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,333 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4] 2024-11-14T09:50:28,334 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f95f44af18143488d0e9344397583aa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:28,334 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5de5ca811bd4359b5691061118e3843, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:28,334 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 5242cb04e37341b5bae0df9d2cca144f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731577825929 2024-11-14T09:50:28,335 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27b17721b6794ef3baf68c96836432e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731577825929 2024-11-14T09:50:28,335 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de50e11e71646169ca2061f73085886, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828157 2024-11-14T09:50:28,335 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d2494324b444299521b0de26eb09d4, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828149 2024-11-14T09:50:28,346 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:28,346 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1554e1dc5c4f4200bc6989c818341ae6 is 50, key is test_row_0/B:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,349 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:28,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:28,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T09:50:28,377 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114bd36bbb916194ce7b2b2f0be68b63eae_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,379 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114bd36bbb916194ce7b2b2f0be68b63eae_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,379 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bd36bbb916194ce7b2b2f0be68b63eae_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577888380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577888385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742430_1606 (size=12831) 2024-11-14T09:50:28,403 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1554e1dc5c4f4200bc6989c818341ae6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1554e1dc5c4f4200bc6989c818341ae6 2024-11-14T09:50:28,409 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into 1554e1dc5c4f4200bc6989c818341ae6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:28,409 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,409 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=13, startTime=1731577828331; duration=0sec 2024-11-14T09:50:28,409 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:28,409 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:28,409 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:28,411 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:28,411 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:28,411 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,411 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/d24dfd81a7c5448280ec26e53dd11c9f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=36.2 K 2024-11-14T09:50:28,412 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d24dfd81a7c5448280ec26e53dd11c9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731577825844 2024-11-14T09:50:28,413 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 462455b5716147c78d85d9994dedb828, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731577825929 2024-11-14T09:50:28,413 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 32da59376e7346d9beadf7277b856c4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828157 2024-11-14T09:50:28,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141e8c18e6a3624ed5908f6611a2ba92e2_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577828243/Put/seqid=0 2024-11-14T09:50:28,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:28,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:28,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,442 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:28,443 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/3d4add897cbe4d20940ad69e1c5b28e2 is 50, key is test_row_0/C:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742431_1607 (size=4469) 2024-11-14T09:50:28,450 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#517 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:28,451 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/bb0e72b57e754208b80768a9bf9e6811 is 175, key is test_row_0/A:col10/1731577828162/Put/seqid=0 2024-11-14T09:50:28,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742432_1608 (size=14994) 2024-11-14T09:50:28,478 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141e8c18e6a3624ed5908f6611a2ba92e2_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141e8c18e6a3624ed5908f6611a2ba92e2_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:28,482 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a9889a55f2aa4e7fad92f5b40f2b35f7, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:28,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a9889a55f2aa4e7fad92f5b40f2b35f7 is 175, key is test_row_0/A:col10/1731577828243/Put/seqid=0 2024-11-14T09:50:28,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742433_1609 (size=12831) 2024-11-14T09:50:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577888494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577888495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,502 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/3d4add897cbe4d20940ad69e1c5b28e2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/3d4add897cbe4d20940ad69e1c5b28e2 2024-11-14T09:50:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742434_1610 (size=31785) 2024-11-14T09:50:28,508 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into 3d4add897cbe4d20940ad69e1c5b28e2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:28,508 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,508 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=13, startTime=1731577828331; duration=0sec 2024-11-14T09:50:28,508 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:28,508 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:28,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742435_1611 (size=39949) 2024-11-14T09:50:28,525 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a9889a55f2aa4e7fad92f5b40f2b35f7 2024-11-14T09:50:28,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/f5c81500f2fd47d49f7a981e8eda9ffd is 50, key is test_row_0/B:col10/1731577828243/Put/seqid=0 2024-11-14T09:50:28,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742436_1612 (size=12301) 2024-11-14T09:50:28,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/f5c81500f2fd47d49f7a981e8eda9ffd 2024-11-14T09:50:28,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/c207126c8cd8453e9b0774e5b8eb5686 is 50, key is test_row_0/C:col10/1731577828243/Put/seqid=0 2024-11-14T09:50:28,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742437_1613 (size=12301) 2024-11-14T09:50:28,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/c207126c8cd8453e9b0774e5b8eb5686 2024-11-14T09:50:28,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/a9889a55f2aa4e7fad92f5b40f2b35f7 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7 2024-11-14T09:50:28,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7, entries=200, sequenceid=284, filesize=39.0 K 2024-11-14T09:50:28,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/f5c81500f2fd47d49f7a981e8eda9ffd as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd 2024-11-14T09:50:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T09:50:28,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/c207126c8cd8453e9b0774e5b8eb5686 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686 2024-11-14T09:50:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686, entries=150, sequenceid=284, filesize=12.0 K 2024-11-14T09:50:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e6b85e37a52d10a57579fd75d18d566c in 226ms, sequenceid=284, compaction requested=false 2024-11-14T09:50:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:28,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465b8e3eab4ee45b8b68453232ce0d317_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742438_1614 (size=12454) 2024-11-14T09:50:28,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:28,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:28,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577888775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577888779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577888884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:28,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:28,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577888895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:28,912 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/bb0e72b57e754208b80768a9bf9e6811 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811 2024-11-14T09:50:28,927 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into bb0e72b57e754208b80768a9bf9e6811(size=31.0 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:28,927 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:28,927 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=13, startTime=1731577828331; duration=0sec 2024-11-14T09:50:28,927 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:28,927 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:29,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577889094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577889105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,135 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:29,137 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465b8e3eab4ee45b8b68453232ce0d317_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465b8e3eab4ee45b8b68453232ce0d317_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:29,138 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f136d6628f694f1c9dcc9673f6ab7209, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:29,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f136d6628f694f1c9dcc9673f6ab7209 is 175, key is test_row_0/A:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:29,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742439_1615 (size=31255) 2024-11-14T09:50:29,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577889405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:29,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577889414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f136d6628f694f1c9dcc9673f6ab7209 2024-11-14T09:50:29,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1a9716eff1c146668addc429ab74cf3c is 50, key is test_row_0/B:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:29,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742440_1616 (size=12301) 2024-11-14T09:50:29,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577889914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577889925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:29,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:29,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:29,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:29,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:29,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1a9716eff1c146668addc429ab74cf3c 2024-11-14T09:50:29,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/db2893206c814ee1815572b8a1843057 is 50, key is test_row_0/C:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:29,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742441_1617 (size=12301) 2024-11-14T09:50:29,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/db2893206c814ee1815572b8a1843057 2024-11-14T09:50:29,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/f136d6628f694f1c9dcc9673f6ab7209 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209 2024-11-14T09:50:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209, entries=150, sequenceid=297, filesize=30.5 K 2024-11-14T09:50:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1a9716eff1c146668addc429ab74cf3c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c 2024-11-14T09:50:29,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T09:50:29,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/db2893206c814ee1815572b8a1843057 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057 2024-11-14T09:50:29,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T09:50:30,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e6b85e37a52d10a57579fd75d18d566c in 1290ms, sequenceid=297, compaction requested=true 2024-11-14T09:50:30,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:30,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:30,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:30,000 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:30,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:30,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:30,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:30,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:30,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:30,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:30,001 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:30,001 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:30,001 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:30,001 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,001 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,002 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=100.6 K 2024-11-14T09:50:30,002 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1554e1dc5c4f4200bc6989c818341ae6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=36.6 K 2024-11-14T09:50:30,002 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,002 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209] 2024-11-14T09:50:30,002 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1554e1dc5c4f4200bc6989c818341ae6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828157 2024-11-14T09:50:30,002 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0e72b57e754208b80768a9bf9e6811, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828157 2024-11-14T09:50:30,003 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c81500f2fd47d49f7a981e8eda9ffd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577828219 2024-11-14T09:50:30,003 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9889a55f2aa4e7fad92f5b40f2b35f7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577828219 2024-11-14T09:50:30,003 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9716eff1c146668addc429ab74cf3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:30,003 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting f136d6628f694f1c9dcc9673f6ab7209, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:30,007 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,008 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:30,008 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114607542aeb88e4aafa3863e4d980486c5_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,009 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/9fc4daf687544ca4863a71716b1e352e is 50, key is test_row_0/B:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:30,010 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114607542aeb88e4aafa3863e4d980486c5_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,010 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114607542aeb88e4aafa3863e4d980486c5_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,011 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:30,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T09:50:30,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742442_1618 (size=4469) 2024-11-14T09:50:30,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d22c0f2c555749e2b75984caf1b83da9_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577830005/Put/seqid=0 2024-11-14T09:50:30,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742443_1619 (size=12983) 2024-11-14T09:50:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577890029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742444_1620 (size=14994) 2024-11-14T09:50:30,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d22c0f2c555749e2b75984caf1b83da9_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d22c0f2c555749e2b75984caf1b83da9_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577890034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/5b91fc3ce4064165bbc6ca13561fb043, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/5b91fc3ce4064165bbc6ca13561fb043 is 175, key is test_row_0/A:col10/1731577830005/Put/seqid=0 2024-11-14T09:50:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742445_1621 (size=39949) 2024-11-14T09:50:30,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/5b91fc3ce4064165bbc6ca13561fb043 2024-11-14T09:50:30,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36590 deadline: 1731577890055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,056 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/e18200eb2c08465e828dbc0eae164b89 is 50, key is test_row_0/B:col10/1731577830005/Put/seqid=0 2024-11-14T09:50:30,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742446_1622 (size=12301) 2024-11-14T09:50:30,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/e18200eb2c08465e828dbc0eae164b89 2024-11-14T09:50:30,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a5dd81269cb8435c9471ae0e8e339a7c is 50, key is test_row_0/C:col10/1731577830005/Put/seqid=0 2024-11-14T09:50:30,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742447_1623 (size=12301) 2024-11-14T09:50:30,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577890145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577890154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577890355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577890364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:30,418 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:30,419 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/2e3ef14d645443c4b2791798d4c2d25f is 175, key is test_row_0/A:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:30,433 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/9fc4daf687544ca4863a71716b1e352e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9fc4daf687544ca4863a71716b1e352e 2024-11-14T09:50:30,437 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into 9fc4daf687544ca4863a71716b1e352e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:30,437 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:30,437 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=13, startTime=1731577830000; duration=0sec 2024-11-14T09:50:30,437 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:30,437 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:30,437 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T09:50:30,438 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T09:50:30,438 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:30,438 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,438 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/3d4add897cbe4d20940ad69e1c5b28e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=36.6 K 2024-11-14T09:50:30,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4add897cbe4d20940ad69e1c5b28e2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731577828157 2024-11-14T09:50:30,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting c207126c8cd8453e9b0774e5b8eb5686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731577828219 2024-11-14T09:50:30,439 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting db2893206c814ee1815572b8a1843057, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:30,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742448_1624 (size=31937) 2024-11-14T09:50:30,445 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#530 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:30,446 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/77b9a1910a484d5e8b46beb6ba1263c6 is 50, key is test_row_0/C:col10/1731577828376/Put/seqid=0 2024-11-14T09:50:30,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742449_1625 (size=12983) 2024-11-14T09:50:30,465 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/77b9a1910a484d5e8b46beb6ba1263c6 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/77b9a1910a484d5e8b46beb6ba1263c6 2024-11-14T09:50:30,469 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into 77b9a1910a484d5e8b46beb6ba1263c6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:30,469 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:30,469 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=13, startTime=1731577830001; duration=0sec 2024-11-14T09:50:30,469 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:30,469 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:30,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a5dd81269cb8435c9471ae0e8e339a7c 2024-11-14T09:50:30,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/5b91fc3ce4064165bbc6ca13561fb043 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043 2024-11-14T09:50:30,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043, entries=200, sequenceid=324, filesize=39.0 K 2024-11-14T09:50:30,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/e18200eb2c08465e828dbc0eae164b89 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89 2024-11-14T09:50:30,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T09:50:30,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a5dd81269cb8435c9471ae0e8e339a7c as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c 2024-11-14T09:50:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T09:50:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e6b85e37a52d10a57579fd75d18d566c in 511ms, sequenceid=324, compaction requested=false 2024-11-14T09:50:30,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-14T09:50:30,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:30,573 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T09:50:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c603bb9fdc0349a585891afa6bc972b6_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577830028/Put/seqid=0 2024-11-14T09:50:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742450_1626 (size=9914) 2024-11-14T09:50:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,588 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c603bb9fdc0349a585891afa6bc972b6_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c603bb9fdc0349a585891afa6bc972b6_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:30,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/edcfb72021564f2793d55583a34a9603, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:30,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/edcfb72021564f2793d55583a34a9603 is 175, key is test_row_0/A:col10/1731577830028/Put/seqid=0 2024-11-14T09:50:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742451_1627 (size=22561) 2024-11-14T09:50:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,593 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/edcfb72021564f2793d55583a34a9603 2024-11-14T09:50:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/36114834f8d04500a51c1a09442e4cf9 is 50, key is test_row_0/B:col10/1731577830028/Put/seqid=0 2024-11-14T09:50:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742452_1628 (size=9857) 2024-11-14T09:50:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577890758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577890760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,847 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/2e3ef14d645443c4b2791798d4c2d25f as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f 2024-11-14T09:50:30,851 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into 2e3ef14d645443c4b2791798d4c2d25f(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:30,851 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:30,851 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=13, startTime=1731577830000; duration=0sec 2024-11-14T09:50:30,851 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:30,851 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577890875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577890875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1731577890924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,927 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_1' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:30,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36586 deadline: 1731577890934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:30,937 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to defc576eb6b7,40243,1731577658084 for 'test_row_0' in e6b85e37a52d10a57579fd75d18d566c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-14T09:50:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,002 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/36114834f8d04500a51c1a09442e4cf9 2024-11-14T09:50:31,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/24f034131ec84d0dbe15c076a07f02d1 is 50, key is test_row_0/C:col10/1731577830028/Put/seqid=0 2024-11-14T09:50:31,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742453_1629 (size=9857) 2024-11-14T09:50:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/24f034131ec84d0dbe15c076a07f02d1 2024-11-14T09:50:31,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/edcfb72021564f2793d55583a34a9603 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603 2024-11-14T09:50:31,029 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603, entries=100, sequenceid=336, filesize=22.0 K 2024-11-14T09:50:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/36114834f8d04500a51c1a09442e4cf9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9 2024-11-14T09:50:31,034 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9, entries=100, sequenceid=336, filesize=9.6 K 2024-11-14T09:50:31,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/24f034131ec84d0dbe15c076a07f02d1 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1 2024-11-14T09:50:31,038 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1, entries=100, sequenceid=336, filesize=9.6 K 2024-11-14T09:50:31,039 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e6b85e37a52d10a57579fd75d18d566c in 466ms, sequenceid=336, compaction requested=true 2024-11-14T09:50:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-14T09:50:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-14T09:50:31,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T09:50:31,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7650 sec 2024-11-14T09:50:31,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.7700 sec 2024-11-14T09:50:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:31,087 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:31,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T09:50:31,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d2a2dbca7b94c069b1593e83a697c8a_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:31,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577891097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577891099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742454_1630 (size=14994) 2024-11-14T09:50:31,110 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:31,114 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d2a2dbca7b94c069b1593e83a697c8a_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d2a2dbca7b94c069b1593e83a697c8a_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:31,115 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8441fa37f1544a6ea04058e1fbfa7e7a, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:31,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8441fa37f1544a6ea04058e1fbfa7e7a is 175, key is test_row_0/A:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742455_1631 (size=39949) 2024-11-14T09:50:31,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577891204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577891204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577891415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577891415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8441fa37f1544a6ea04058e1fbfa7e7a 2024-11-14T09:50:31,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a is 50, key is test_row_0/B:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:31,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742456_1632 (size=12301) 2024-11-14T09:50:31,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a 2024-11-14T09:50:31,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2a3d5ae3c07e4d9092b3ace5fd375b92 is 50, key is test_row_0/C:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742457_1633 (size=12301) 2024-11-14T09:50:31,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577891724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577891725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:31,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2a3d5ae3c07e4d9092b3ace5fd375b92 2024-11-14T09:50:31,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8441fa37f1544a6ea04058e1fbfa7e7a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a 2024-11-14T09:50:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a, entries=200, sequenceid=365, filesize=39.0 K 2024-11-14T09:50:31,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a 2024-11-14T09:50:31,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T09:50:31,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/2a3d5ae3c07e4d9092b3ace5fd375b92 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92 2024-11-14T09:50:31,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T09:50:31,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for e6b85e37a52d10a57579fd75d18d566c in 878ms, sequenceid=365, compaction requested=true 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:31,965 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6b85e37a52d10a57579fd75d18d566c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T09:50:31,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:31,965 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:31,970 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:31,970 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:31,970 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/A is initiating minor compaction (all files) 2024-11-14T09:50:31,970 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/B is initiating minor compaction (all files) 2024-11-14T09:50:31,970 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/A in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:31,970 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/B in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:31,970 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=131.2 K 2024-11-14T09:50:31,970 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9fc4daf687544ca4863a71716b1e352e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=46.3 K 2024-11-14T09:50:31,970 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:31,970 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. files: [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a] 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc4daf687544ca4863a71716b1e352e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e3ef14d645443c4b2791798d4c2d25f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b91fc3ce4064165bbc6ca13561fb043, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577828760 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting e18200eb2c08465e828dbc0eae164b89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577828760 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting edcfb72021564f2793d55583a34a9603, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731577830028 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 36114834f8d04500a51c1a09442e4cf9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731577830028 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8441fa37f1544a6ea04058e1fbfa7e7a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731577830750 2024-11-14T09:50:31,971 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting d0f7c1dd3f6441c8b4473d40ac8f9d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731577830750 2024-11-14T09:50:31,981 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#B#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:31,982 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1632ffa53c43402d9a2b4032150fa939 is 50, key is test_row_0/B:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:31,985 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,001 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114e942760bf47e4a57a653fe68d65d0e3c_e6b85e37a52d10a57579fd75d18d566c store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,003 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114e942760bf47e4a57a653fe68d65d0e3c_e6b85e37a52d10a57579fd75d18d566c, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,003 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e942760bf47e4a57a653fe68d65d0e3c_e6b85e37a52d10a57579fd75d18d566c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742458_1634 (size=13119) 2024-11-14T09:50:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742459_1635 (size=4469) 2024-11-14T09:50:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:32,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T09:50:32,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T09:50:32,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e626ce185a418ba41b6d9acb4c9e44_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577831098/Put/seqid=0 2024-11-14T09:50:32,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742460_1636 (size=17534) 2024-11-14T09:50:32,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577892297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577892302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577892404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,409 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/1632ffa53c43402d9a2b4032150fa939 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1632ffa53c43402d9a2b4032150fa939 2024-11-14T09:50:32,412 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/B of e6b85e37a52d10a57579fd75d18d566c into 1632ffa53c43402d9a2b4032150fa939(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:32,412 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:32,412 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/B, priority=12, startTime=1731577831965; duration=0sec 2024-11-14T09:50:32,412 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T09:50:32,412 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:B 2024-11-14T09:50:32,412 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T09:50:32,413 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T09:50:32,413 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1541): e6b85e37a52d10a57579fd75d18d566c/C is initiating minor compaction (all files) 2024-11-14T09:50:32,413 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6b85e37a52d10a57579fd75d18d566c/C in TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:32,413 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/77b9a1910a484d5e8b46beb6ba1263c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92] into tmpdir=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp, totalSize=46.3 K 2024-11-14T09:50:32,414 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b9a1910a484d5e8b46beb6ba1263c6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731577828375 2024-11-14T09:50:32,414 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting a5dd81269cb8435c9471ae0e8e339a7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731577828760 2024-11-14T09:50:32,414 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f034131ec84d0dbe15c076a07f02d1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731577830028 2024-11-14T09:50:32,414 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3d5ae3c07e4d9092b3ace5fd375b92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731577830750 2024-11-14T09:50:32,415 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#A#compaction#538 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:32,415 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/c44b44499d4c46c69f8f3577a045a8f9 is 175, key is test_row_0/A:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:32,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577892414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742461_1637 (size=32073) 2024-11-14T09:50:32,421 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6b85e37a52d10a57579fd75d18d566c#C#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T09:50:32,422 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/f780c444b0684141a532a2290c415d95 is 50, key is test_row_0/C:col10/1731577830750/Put/seqid=0 2024-11-14T09:50:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742462_1638 (size=13119) 2024-11-14T09:50:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-14T09:50:32,435 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:32,436 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T09:50:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-14T09:50:32,437 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T09:50:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:32,438 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T09:50:32,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T09:50:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:32,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T09:50:32,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:32,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:32,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T09:50:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36522 deadline: 1731577892614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T09:50:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36546 deadline: 1731577892625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 , the old value is region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T09:50:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., hostname=defc576eb6b7,40243,1731577658084, seqNum=5 because the exception is null or not the one we care about 2024-11-14T09:50:32,651 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:32,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c4e626ce185a418ba41b6d9acb4c9e44_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e626ce185a418ba41b6d9acb4c9e44_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:32,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8e3c4cd9216c4060a230f77b90d4b882, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8e3c4cd9216c4060a230f77b90d4b882 is 175, key is test_row_0/A:col10/1731577831098/Put/seqid=0 2024-11-14T09:50:32,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742463_1639 (size=48639) 2024-11-14T09:50:32,662 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8e3c4cd9216c4060a230f77b90d4b882 2024-11-14T09:50:32,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/709d0321a37249c1842f9fc8e8e2c918 is 50, key is test_row_0/B:col10/1731577831098/Put/seqid=0 2024-11-14T09:50:32,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742464_1640 (size=12301) 2024-11-14T09:50:32,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/709d0321a37249c1842f9fc8e8e2c918 2024-11-14T09:50:32,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a136cabb24b84df09bd8ca75af095a22 is 50, key is test_row_0/C:col10/1731577831098/Put/seqid=0 2024-11-14T09:50:32,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742465_1641 (size=12301) 2024-11-14T09:50:32,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a136cabb24b84df09bd8ca75af095a22 2024-11-14T09:50:32,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/8e3c4cd9216c4060a230f77b90d4b882 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8e3c4cd9216c4060a230f77b90d4b882 2024-11-14T09:50:32,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8e3c4cd9216c4060a230f77b90d4b882, entries=250, sequenceid=377, filesize=47.5 K 2024-11-14T09:50:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/709d0321a37249c1842f9fc8e8e2c918 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/709d0321a37249c1842f9fc8e8e2c918 2024-11-14T09:50:32,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/709d0321a37249c1842f9fc8e8e2c918, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T09:50:32,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/a136cabb24b84df09bd8ca75af095a22 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a136cabb24b84df09bd8ca75af095a22 2024-11-14T09:50:32,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a136cabb24b84df09bd8ca75af095a22, entries=150, sequenceid=377, filesize=12.0 K 2024-11-14T09:50:32,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e6b85e37a52d10a57579fd75d18d566c in 485ms, sequenceid=377, compaction requested=false 2024-11-14T09:50:32,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:32,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40243 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-14T09:50:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:32,742 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T09:50:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111440b664e0442f4677a9bc60dc525fa9ce_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577832296/Put/seqid=0 2024-11-14T09:50:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742466_1642 (size=12454) 2024-11-14T09:50:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:32,752 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111440b664e0442f4677a9bc60dc525fa9ce_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440b664e0442f4677a9bc60dc525fa9ce_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/01624f561e914c7aaf46bcd45e5bda8e, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/01624f561e914c7aaf46bcd45e5bda8e is 175, key is test_row_0/A:col10/1731577832296/Put/seqid=0 2024-11-14T09:50:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742467_1643 (size=31255) 2024-11-14T09:50:32,824 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/c44b44499d4c46c69f8f3577a045a8f9 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/c44b44499d4c46c69f8f3577a045a8f9 2024-11-14T09:50:32,828 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/A of e6b85e37a52d10a57579fd75d18d566c into c44b44499d4c46c69f8f3577a045a8f9(size=31.3 K), total size for store is 78.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:32,828 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:32,828 INFO [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/A, priority=12, startTime=1731577831965; duration=0sec 2024-11-14T09:50:32,828 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:32,828 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/f780c444b0684141a532a2290c415d95 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/f780c444b0684141a532a2290c415d95 2024-11-14T09:50:32,828 DEBUG [RS:0;defc576eb6b7:40243-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:A 2024-11-14T09:50:32,832 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6b85e37a52d10a57579fd75d18d566c/C of e6b85e37a52d10a57579fd75d18d566c into f780c444b0684141a532a2290c415d95(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T09:50:32,832 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:32,832 INFO [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., storeName=e6b85e37a52d10a57579fd75d18d566c/C, priority=12, startTime=1731577831965; duration=0sec 2024-11-14T09:50:32,832 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T09:50:32,833 DEBUG [RS:0;defc576eb6b7:40243-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6b85e37a52d10a57579fd75d18d566c:C 2024-11-14T09:50:32,902 INFO [Thread-2409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2409. 2024-11-14T09:50:32,902 DEBUG [Thread-2409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,902 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,902 DEBUG [Thread-2409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,903 INFO [Thread-2403 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2403. 2024-11-14T09:50:32,903 DEBUG [Thread-2403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,903 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,903 DEBUG [Thread-2403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,903 INFO [Thread-2405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2405. 2024-11-14T09:50:32,903 DEBUG [Thread-2405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,903 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,903 DEBUG [Thread-2405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,905 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-14T09:50:32,905 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,905 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,905 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,905 INFO [Thread-2407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2407. 2024-11-14T09:50:32,905 DEBUG [Thread-2407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,905 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,905 DEBUG [Thread-2407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40243 {}] regionserver.HRegion(8855): Flush requested on e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:32,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. as already flushing 2024-11-14T09:50:32,927 INFO [Thread-2392 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2392. 2024-11-14T09:50:32,927 DEBUG [Thread-2392 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,927 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,927 DEBUG [Thread-2392 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,936 INFO [Thread-2398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2398. 2024-11-14T09:50:32,936 DEBUG [Thread-2398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,936 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,936 DEBUG [Thread-2398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,946 INFO [Thread-2394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2394. 2024-11-14T09:50:32,946 DEBUG [Thread-2394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,946 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,946 DEBUG [Thread-2394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,946 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:32,967 INFO [Thread-2390 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2390. 2024-11-14T09:50:32,967 DEBUG [Thread-2390 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:32,967 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,967 DEBUG [Thread-2390 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:32,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:33,157 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/01624f561e914c7aaf46bcd45e5bda8e 2024-11-14T09:50:33,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2ee93afddcdd4cf2a65373f3fcd8a467 is 50, key is test_row_0/B:col10/1731577832296/Put/seqid=0 2024-11-14T09:50:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742468_1644 (size=12301) 2024-11-14T09:50:33,565 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2ee93afddcdd4cf2a65373f3fcd8a467 2024-11-14T09:50:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/ae40705059894d2bb1958ad3a18f3909 is 50, key is test_row_0/C:col10/1731577832296/Put/seqid=0 2024-11-14T09:50:33,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742469_1645 (size=12301) 2024-11-14T09:50:33,979 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/ae40705059894d2bb1958ad3a18f3909 2024-11-14T09:50:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/01624f561e914c7aaf46bcd45e5bda8e as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/01624f561e914c7aaf46bcd45e5bda8e 2024-11-14T09:50:33,989 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/01624f561e914c7aaf46bcd45e5bda8e, entries=150, sequenceid=402, filesize=30.5 K 2024-11-14T09:50:33,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/2ee93afddcdd4cf2a65373f3fcd8a467 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2ee93afddcdd4cf2a65373f3fcd8a467 2024-11-14T09:50:33,993 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2ee93afddcdd4cf2a65373f3fcd8a467, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T09:50:33,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/ae40705059894d2bb1958ad3a18f3909 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/ae40705059894d2bb1958ad3a18f3909 2024-11-14T09:50:33,996 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/ae40705059894d2bb1958ad3a18f3909, entries=150, sequenceid=402, filesize=12.0 K 2024-11-14T09:50:33,997 INFO [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for e6b85e37a52d10a57579fd75d18d566c in 1255ms, sequenceid=402, compaction requested=true 2024-11-14T09:50:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for e6b85e37a52d10a57579fd75d18d566c: 2024-11-14T09:50:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/defc576eb6b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-14T09:50:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-14T09:50:33,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-14T09:50:33,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-14T09:50:34,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.5630 sec 2024-11-14T09:50:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T09:50:34,585 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:36,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T09:50:40,086 INFO [Thread-2396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2396. 2024-11-14T09:50:40,086 DEBUG [Thread-2396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T09:50:40,086 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:40,087 DEBUG [Thread-2396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:40,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3594 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3418 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3460 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3603 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3438 2024-11-14T09:50:40,087 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T09:50:40,087 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:50:40,087 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:50:40,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:40,087 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:40,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:40,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T09:50:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:40,091 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577840091"}]},"ts":"1731577840091"} 2024-11-14T09:50:40,093 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T09:50:40,093 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T09:50:40,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T09:50:40,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, UNASSIGN}] 2024-11-14T09:50:40,095 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, UNASSIGN 2024-11-14T09:50:40,096 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=CLOSING, regionLocation=defc576eb6b7,40243,1731577658084 2024-11-14T09:50:40,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, UNASSIGN because future has completed 2024-11-14T09:50:40,098 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T09:50:40,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084}] 2024-11-14T09:50:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:40,250 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:40,250 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T09:50:40,250 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing e6b85e37a52d10a57579fd75d18d566c, disabling compactions & flushes 2024-11-14T09:50:40,250 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:40,250 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:40,250 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. after waiting 0 ms 2024-11-14T09:50:40,250 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:40,250 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing e6b85e37a52d10a57579fd75d18d566c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T09:50:40,254 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b08e35a4410f47d4a5939a88877cf143_e6b85e37a52d10a57579fd75d18d566c is 50, key is test_row_0/A:col10/1731577832965/Put/seqid=0 2024-11-14T09:50:40,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742470_1646 (size=12454) 2024-11-14T09:50:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:40,658 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T09:50:40,661 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b08e35a4410f47d4a5939a88877cf143_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b08e35a4410f47d4a5939a88877cf143_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:40,662 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/0fd435c16ac5474cb172f55f080f959d, store: [table=TestAcidGuarantees family=A region=e6b85e37a52d10a57579fd75d18d566c] 2024-11-14T09:50:40,662 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/0fd435c16ac5474cb172f55f080f959d is 175, key is test_row_0/A:col10/1731577832965/Put/seqid=0 2024-11-14T09:50:40,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742471_1647 (size=31255) 2024-11-14T09:50:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:41,065 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=412, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/0fd435c16ac5474cb172f55f080f959d 2024-11-14T09:50:41,071 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/278e0583f5854e67b860adeb5031809d is 50, key is test_row_0/B:col10/1731577832965/Put/seqid=0 2024-11-14T09:50:41,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742472_1648 (size=12301) 2024-11-14T09:50:41,079 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/278e0583f5854e67b860adeb5031809d 2024-11-14T09:50:41,085 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6fefcd120e7640debf8d40bd99ccdddc is 50, key is test_row_0/C:col10/1731577832965/Put/seqid=0 2024-11-14T09:50:41,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742473_1649 (size=12301) 2024-11-14T09:50:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:41,489 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6fefcd120e7640debf8d40bd99ccdddc 2024-11-14T09:50:41,492 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/A/0fd435c16ac5474cb172f55f080f959d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/0fd435c16ac5474cb172f55f080f959d 2024-11-14T09:50:41,495 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/0fd435c16ac5474cb172f55f080f959d, entries=150, sequenceid=412, filesize=30.5 K 2024-11-14T09:50:41,495 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/B/278e0583f5854e67b860adeb5031809d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/278e0583f5854e67b860adeb5031809d 2024-11-14T09:50:41,498 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/278e0583f5854e67b860adeb5031809d, entries=150, sequenceid=412, filesize=12.0 K 2024-11-14T09:50:41,498 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/.tmp/C/6fefcd120e7640debf8d40bd99ccdddc as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6fefcd120e7640debf8d40bd99ccdddc 2024-11-14T09:50:41,501 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6fefcd120e7640debf8d40bd99ccdddc, entries=150, sequenceid=412, filesize=12.0 K 2024-11-14T09:50:41,502 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e6b85e37a52d10a57579fd75d18d566c in 1251ms, sequenceid=412, compaction requested=true 2024-11-14T09:50:41,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a] to archive 2024-11-14T09:50:41,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:41,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4e98eecbf1ac4770a6a836fa89ca0433 2024-11-14T09:50:41,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/77de1e16fae14992940a66e235c93382 2024-11-14T09:50:41,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/428975ec1c344bb99e3fe4bf896a0fab 2024-11-14T09:50:41,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a5f14be76af44b9f9384e38530bf8569 2024-11-14T09:50:41,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/e1bcf7d1199e448a92b9d2ce1039d87c 2024-11-14T09:50:41,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2367232e7b0f49d187faafa88f14342c 2024-11-14T09:50:41,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/4b09cc55472e49f28cf92a7c8b673141 2024-11-14T09:50:41,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/150ace4093d349f38eb459c06d4dea27 2024-11-14T09:50:41,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/398e64ee2eee480ba1b58d681a7b7c4b 2024-11-14T09:50:41,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f98d5ff4f0524d57a3b8cc5eb8601736 2024-11-14T09:50:41,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/731777b830e24da0803050c3ffe34e40 2024-11-14T09:50:41,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/3669283c7be5455a95f1fc22c8722721 2024-11-14T09:50:41,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/633b5d438ef344d09dbb8ce198ca058c 2024-11-14T09:50:41,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/ecdeb18d372240c1b24985ba38c3ece7 2024-11-14T09:50:41,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/b5de5ca811bd4359b5691061118e3843 2024-11-14T09:50:41,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/27b17721b6794ef3baf68c96836432e8 2024-11-14T09:50:41,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/85d2494324b444299521b0de26eb09d4 2024-11-14T09:50:41,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/bb0e72b57e754208b80768a9bf9e6811 2024-11-14T09:50:41,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/a9889a55f2aa4e7fad92f5b40f2b35f7 2024-11-14T09:50:41,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/2e3ef14d645443c4b2791798d4c2d25f 2024-11-14T09:50:41,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/f136d6628f694f1c9dcc9673f6ab7209 2024-11-14T09:50:41,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/5b91fc3ce4064165bbc6ca13561fb043 2024-11-14T09:50:41,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/edcfb72021564f2793d55583a34a9603 2024-11-14T09:50:41,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8441fa37f1544a6ea04058e1fbfa7e7a 2024-11-14T09:50:41,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/db680d9f6b584b458bb79d7a9b888c02, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f12975dedb6443a1b24545b02c0abb44, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5f95f44af18143488d0e9344397583aa, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1554e1dc5c4f4200bc6989c818341ae6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9fc4daf687544ca4863a71716b1e352e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a] to archive 2024-11-14T09:50:41,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:41,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/7e296c7ade8f4dbd80fb211043bef9cf 2024-11-14T09:50:41,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2f4a90a988f4409a9a8738f823721de5 2024-11-14T09:50:41,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0b728aa1a2cb4e25820ddf9c4c615c6e 2024-11-14T09:50:41,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/93f6f4e5572b4ed8a1e1de0ff46ef851 2024-11-14T09:50:41,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/44570101b7b5488c95eb2da8f890b6a5 2024-11-14T09:50:41,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/db680d9f6b584b458bb79d7a9b888c02 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/db680d9f6b584b458bb79d7a9b888c02 2024-11-14T09:50:41,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c175133ad1954163a9622465c489cb99 2024-11-14T09:50:41,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/c38564a4c1a8458e8c841398ccb9610d 2024-11-14T09:50:41,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/50ef2b3055124665b68b28ec23b9ce0f 2024-11-14T09:50:41,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f12975dedb6443a1b24545b02c0abb44 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f12975dedb6443a1b24545b02c0abb44 2024-11-14T09:50:41,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/086b4929189843d5a6d3acc633c789c9 2024-11-14T09:50:41,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/df9920ac0067494b8463640537e35822 2024-11-14T09:50:41,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9b6eb375d2974c438a596af140bd9176 2024-11-14T09:50:41,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5f95f44af18143488d0e9344397583aa to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5f95f44af18143488d0e9344397583aa 2024-11-14T09:50:41,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/750b760ed4814229aac63f30dbb488da 2024-11-14T09:50:41,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/5242cb04e37341b5bae0df9d2cca144f 2024-11-14T09:50:41,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1554e1dc5c4f4200bc6989c818341ae6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1554e1dc5c4f4200bc6989c818341ae6 2024-11-14T09:50:41,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/0de50e11e71646169ca2061f73085886 2024-11-14T09:50:41,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/f5c81500f2fd47d49f7a981e8eda9ffd 2024-11-14T09:50:41,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9fc4daf687544ca4863a71716b1e352e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/9fc4daf687544ca4863a71716b1e352e 2024-11-14T09:50:41,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1a9716eff1c146668addc429ab74cf3c 2024-11-14T09:50:41,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/e18200eb2c08465e828dbc0eae164b89 2024-11-14T09:50:41,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/36114834f8d04500a51c1a09442e4cf9 2024-11-14T09:50:41,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/d0f7c1dd3f6441c8b4473d40ac8f9d1a 2024-11-14T09:50:41,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/b45d6451768b4b218c15dece019bc4a4, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a92d10fac32d4668b5e4f607a3a0f636, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/d24dfd81a7c5448280ec26e53dd11c9f, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/3d4add897cbe4d20940ad69e1c5b28e2, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/77b9a1910a484d5e8b46beb6ba1263c6, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92] to archive 2024-11-14T09:50:41,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T09:50:41,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/86cf6b94a0c948b5a2ce2dcea9a837e7 2024-11-14T09:50:41,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a4f87fce6087499985a33212dc5f1671 2024-11-14T09:50:41,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6e246a13e6ef459d8ed213f2a7c934ad 2024-11-14T09:50:41,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32eba3e202da40919bacc9207e0fb77c 2024-11-14T09:50:41,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/41497f95a272423c995ba34ef6f1cf66 2024-11-14T09:50:41,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/b45d6451768b4b218c15dece019bc4a4 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/b45d6451768b4b218c15dece019bc4a4 2024-11-14T09:50:41,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/8f150bc56a794de489319bad944617e3 2024-11-14T09:50:41,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/229a1c9e1f434b6bbe96d7d3552de014 2024-11-14T09:50:41,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/4874c46652eb4982ac1cdca5fbd5b381 2024-11-14T09:50:41,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a92d10fac32d4668b5e4f607a3a0f636 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a92d10fac32d4668b5e4f607a3a0f636 2024-11-14T09:50:41,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/01b07ba3a077435aa8cbe3337ad6e86c 2024-11-14T09:50:41,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2320854061ac4528b4c929857b8e8160 2024-11-14T09:50:41,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/fbe9577bcc014784b78d3890cc89af5e 2024-11-14T09:50:41,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/d24dfd81a7c5448280ec26e53dd11c9f to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/d24dfd81a7c5448280ec26e53dd11c9f 2024-11-14T09:50:41,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/cef638abc2d441aa8818957acf5578a8 2024-11-14T09:50:41,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/462455b5716147c78d85d9994dedb828 2024-11-14T09:50:41,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/3d4add897cbe4d20940ad69e1c5b28e2 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/3d4add897cbe4d20940ad69e1c5b28e2 2024-11-14T09:50:41,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/32da59376e7346d9beadf7277b856c4d 2024-11-14T09:50:41,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/c207126c8cd8453e9b0774e5b8eb5686 2024-11-14T09:50:41,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/77b9a1910a484d5e8b46beb6ba1263c6 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/77b9a1910a484d5e8b46beb6ba1263c6 2024-11-14T09:50:41,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/db2893206c814ee1815572b8a1843057 2024-11-14T09:50:41,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a5dd81269cb8435c9471ae0e8e339a7c 2024-11-14T09:50:41,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/24f034131ec84d0dbe15c076a07f02d1 2024-11-14T09:50:41,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/2a3d5ae3c07e4d9092b3ace5fd375b92 2024-11-14T09:50:41,572 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits/415.seqid, newMaxSeqId=415, maxSeqId=4 2024-11-14T09:50:41,572 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. 2024-11-14T09:50:41,572 DEBUG [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for e6b85e37a52d10a57579fd75d18d566c: Waiting for close lock at 1731577840250Running coprocessor pre-close hooks at 1731577840250Disabling compacts and flushes for region at 1731577840250Disabling writes for close at 1731577840250Obtaining lock to block concurrent updates at 1731577840250Preparing flush snapshotting stores in e6b85e37a52d10a57579fd75d18d566c at 1731577840250Finished memstore snapshotting TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731577840250Flushing stores of TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c. at 1731577840251 (+1 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/A: creating writer at 1731577840251Flushing mob file e6b85e37a52d10a57579fd75d18d566c/A: appending metadata at 1731577840254 (+3 ms)Flushing mob file e6b85e37a52d10a57579fd75d18d566c/A: closing flushed file at 1731577840254Flushing e6b85e37a52d10a57579fd75d18d566c/A: appending metadata at 1731577840662 (+408 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/A: closing flushed file at 1731577840662Flushing e6b85e37a52d10a57579fd75d18d566c/B: creating writer at 1731577841068 (+406 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/B: appending metadata at 1731577841070 (+2 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/B: closing flushed file at 1731577841070Flushing e6b85e37a52d10a57579fd75d18d566c/C: creating writer at 1731577841083 (+13 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/C: appending metadata at 1731577841085 (+2 ms)Flushing e6b85e37a52d10a57579fd75d18d566c/C: closing flushed file at 1731577841085Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fbd7b53: reopening flushed file at 1731577841491 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40c8f092: reopening flushed file at 1731577841495 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d1a9838: reopening flushed file at 1731577841498 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e6b85e37a52d10a57579fd75d18d566c in 1251ms, sequenceid=412, compaction requested=true at 1731577841502 (+4 ms)Writing region close event to WAL at 1731577841570 (+68 ms)Running coprocessor post-close hooks at 1731577841572 (+2 ms)Closed at 1731577841572 2024-11-14T09:50:41,574 INFO [RS_CLOSE_REGION-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:41,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=e6b85e37a52d10a57579fd75d18d566c, regionState=CLOSED 2024-11-14T09:50:41,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 because future has completed 2024-11-14T09:50:41,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-14T09:50:41,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure e6b85e37a52d10a57579fd75d18d566c, server=defc576eb6b7,40243,1731577658084 in 1.4780 sec 2024-11-14T09:50:41,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-14T09:50:41,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6b85e37a52d10a57579fd75d18d566c, UNASSIGN in 1.4820 sec 2024-11-14T09:50:41,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-14T09:50:41,580 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4860 sec 2024-11-14T09:50:41,581 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731577841581"}]},"ts":"1731577841581"} 2024-11-14T09:50:41,582 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T09:50:41,582 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T09:50:41,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4940 sec 2024-11-14T09:50:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-14T09:50:42,235 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:42,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T09:50:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,238 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T09:50:42,239 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,240 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,243 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C, FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits] 2024-11-14T09:50:42,245 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/01624f561e914c7aaf46bcd45e5bda8e to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/01624f561e914c7aaf46bcd45e5bda8e 2024-11-14T09:50:42,246 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/0fd435c16ac5474cb172f55f080f959d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/0fd435c16ac5474cb172f55f080f959d 2024-11-14T09:50:42,247 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8e3c4cd9216c4060a230f77b90d4b882 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/8e3c4cd9216c4060a230f77b90d4b882 2024-11-14T09:50:42,249 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/c44b44499d4c46c69f8f3577a045a8f9 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/A/c44b44499d4c46c69f8f3577a045a8f9 2024-11-14T09:50:42,251 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1632ffa53c43402d9a2b4032150fa939 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/1632ffa53c43402d9a2b4032150fa939 2024-11-14T09:50:42,252 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/278e0583f5854e67b860adeb5031809d to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/278e0583f5854e67b860adeb5031809d 2024-11-14T09:50:42,254 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2ee93afddcdd4cf2a65373f3fcd8a467 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/2ee93afddcdd4cf2a65373f3fcd8a467 2024-11-14T09:50:42,255 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/709d0321a37249c1842f9fc8e8e2c918 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/B/709d0321a37249c1842f9fc8e8e2c918 2024-11-14T09:50:42,257 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6fefcd120e7640debf8d40bd99ccdddc to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/6fefcd120e7640debf8d40bd99ccdddc 2024-11-14T09:50:42,258 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a136cabb24b84df09bd8ca75af095a22 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/a136cabb24b84df09bd8ca75af095a22 2024-11-14T09:50:42,260 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/ae40705059894d2bb1958ad3a18f3909 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/ae40705059894d2bb1958ad3a18f3909 2024-11-14T09:50:42,261 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/f780c444b0684141a532a2290c415d95 to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/C/f780c444b0684141a532a2290c415d95 2024-11-14T09:50:42,263 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits/415.seqid to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c/recovered.edits/415.seqid 2024-11-14T09:50:42,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/default/TestAcidGuarantees/e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,264 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T09:50:42,264 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:50:42,265 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T09:50:42,269 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071ff7c1bebc4decbe10f6673589d631_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114071ff7c1bebc4decbe10f6673589d631_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,270 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e3f93f51359414ca50ca3c193afbeaa_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e3f93f51359414ca50ca3c193afbeaa_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,272 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114182e161025f1414289bee86180283697_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114182e161025f1414289bee86180283697_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,273 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141e8c18e6a3624ed5908f6611a2ba92e2_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141e8c18e6a3624ed5908f6611a2ba92e2_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,275 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142f189709dca543fe9bc28d9bfd4be45c_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142f189709dca543fe9bc28d9bfd4be45c_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,276 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440b664e0442f4677a9bc60dc525fa9ce_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111440b664e0442f4677a9bc60dc525fa9ce_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,278 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144522ac36e50d4744a468b9109ddc50e3_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144522ac36e50d4744a468b9109ddc50e3_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,279 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449505153bb4849249cf413f9eb833085_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111449505153bb4849249cf413f9eb833085_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,280 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d2a2dbca7b94c069b1593e83a697c8a_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d2a2dbca7b94c069b1593e83a697c8a_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,282 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114511281341a6b4f5e82b29b01c8d32f95_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114511281341a6b4f5e82b29b01c8d32f95_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,284 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465b8e3eab4ee45b8b68453232ce0d317_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465b8e3eab4ee45b8b68453232ce0d317_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469ce85f2388f417cb9a15ed268e79f74_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111469ce85f2388f417cb9a15ed268e79f74_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146c4619dba7f345d8bccfa146f2e2f10e_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146c4619dba7f345d8bccfa146f2e2f10e_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,288 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146fba755a07714bab99f0fe06e025432d_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146fba755a07714bab99f0fe06e025432d_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,289 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d0b023491b1463ca61267419cd530ff_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147d0b023491b1463ca61267419cd530ff_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,290 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab82e30f552841de8d8d9ff96d26a583_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ab82e30f552841de8d8d9ff96d26a583_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b08e35a4410f47d4a5939a88877cf143_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b08e35a4410f47d4a5939a88877cf143_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,292 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e626ce185a418ba41b6d9acb4c9e44_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c4e626ce185a418ba41b6d9acb4c9e44_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c603bb9fdc0349a585891afa6bc972b6_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c603bb9fdc0349a585891afa6bc972b6_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d22c0f2c555749e2b75984caf1b83da9_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d22c0f2c555749e2b75984caf1b83da9_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8ec27edc8884f3abbefa60245610c7c_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f8ec27edc8884f3abbefa60245610c7c_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fab25efab2864406b1d194bde2136837_e6b85e37a52d10a57579fd75d18d566c to hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fab25efab2864406b1d194bde2136837_e6b85e37a52d10a57579fd75d18d566c 2024-11-14T09:50:42,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T09:50:42,298 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,300 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T09:50:42,302 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T09:50:42,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,303 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T09:50:42,303 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731577842303"}]},"ts":"9223372036854775807"} 2024-11-14T09:50:42,304 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T09:50:42,304 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e6b85e37a52d10a57579fd75d18d566c, NAME => 'TestAcidGuarantees,,1731577809774.e6b85e37a52d10a57579fd75d18d566c.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T09:50:42,304 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T09:50:42,304 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731577842304"}]},"ts":"9223372036854775807"} 2024-11-14T09:50:42,306 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T09:50:42,306 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T09:50:42,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-14T09:50:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35261 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-14T09:50:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T09:50:42,346 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T09:50:42,361 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=485 (was 506), ProcessCount=11 (was 12), AvailableMemoryMB=4371 (was 4501) 2024-11-14T09:50:42,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T09:50:42,361 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:50:42,361 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:50:42,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:42,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:42,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T09:50:42,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T09:50:42,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-14T09:50:42,362 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=defc576eb6b7,35261,1731577657311 2024-11-14T09:50:42,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:50:42,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T09:50:42,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:50:42,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:50:42,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:50:42,388 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:50:42,388 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T09:50:42,388 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T09:50:42,388 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:50:42,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:42,389 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'defc576eb6b7,40243,1731577658084' ***** 2024-11-14T09:50:42,389 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T09:50:42,390 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T09:50:42,390 INFO [RS:0;defc576eb6b7:40243 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T09:50:42,390 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(959): stopping server defc576eb6b7,40243,1731577658084 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;defc576eb6b7:40243. 2024-11-14T09:50:42,391 DEBUG [RS:0;defc576eb6b7:40243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T09:50:42,391 DEBUG [RS:0;defc576eb6b7:40243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T09:50:42,391 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T09:50:42,392 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T09:50:42,392 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T09:50:42,392 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T09:50:42,393 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:42,393 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T09:50:42,393 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T09:50:42,393 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T09:50:42,393 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T09:50:42,393 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T09:50:42,393 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T09:50:42,413 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/info/7924050775824082a1cd42a6e6fd7cf8 is 95, key is TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e./info:/1731577691005/DeleteFamily/seqid=0 2024-11-14T09:50:42,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742474_1650 (size=6514) 2024-11-14T09:50:42,426 INFO [regionserver/defc576eb6b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:50:42,593 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:42,793 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:42,817 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/info/7924050775824082a1cd42a6e6fd7cf8 2024-11-14T09:50:42,845 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/ns/685845df728247d5891cc9c0d8bf84f4 is 93, key is TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e./ns:/1731577690961/DeleteFamily/seqid=0 2024-11-14T09:50:42,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742475_1651 (size=5906) 2024-11-14T09:50:42,994 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:43,088 INFO [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T09:50:43,089 INFO [regionserver/defc576eb6b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T09:50:43,194 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:43,248 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/ns/685845df728247d5891cc9c0d8bf84f4 2024-11-14T09:50:43,272 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/rep_barrier/fa4cc65d83574b0ba25f2692d4a7a4f2 is 102, key is TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e./rep_barrier:/1731577690961/DeleteFamily/seqid=0 2024-11-14T09:50:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742476_1652 (size=6025) 2024-11-14T09:50:43,394 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:43,594 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:43,676 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/rep_barrier/fa4cc65d83574b0ba25f2692d4a7a4f2 2024-11-14T09:50:43,692 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/table/10470e6cef21449fafdc75676910b57d is 96, key is TestAcidGuarantees,,1731577662821.91a5f3dba40c0b5503a8f1cccd94111e./table:/1731577690961/DeleteFamily/seqid=0 2024-11-14T09:50:43,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742477_1653 (size=5936) 2024-11-14T09:50:43,795 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:43,995 DEBUG [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T09:50:44,096 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/table/10470e6cef21449fafdc75676910b57d 2024-11-14T09:50:44,100 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/info/7924050775824082a1cd42a6e6fd7cf8 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/info/7924050775824082a1cd42a6e6fd7cf8 2024-11-14T09:50:44,103 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/info/7924050775824082a1cd42a6e6fd7cf8, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T09:50:44,103 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/ns/685845df728247d5891cc9c0d8bf84f4 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/ns/685845df728247d5891cc9c0d8bf84f4 2024-11-14T09:50:44,106 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/ns/685845df728247d5891cc9c0d8bf84f4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T09:50:44,107 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/rep_barrier/fa4cc65d83574b0ba25f2692d4a7a4f2 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/rep_barrier/fa4cc65d83574b0ba25f2692d4a7a4f2 2024-11-14T09:50:44,110 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/rep_barrier/fa4cc65d83574b0ba25f2692d4a7a4f2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T09:50:44,110 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/.tmp/table/10470e6cef21449fafdc75676910b57d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/table/10470e6cef21449fafdc75676910b57d 2024-11-14T09:50:44,113 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/table/10470e6cef21449fafdc75676910b57d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T09:50:44,114 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false 2024-11-14T09:50:44,117 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T09:50:44,118 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T09:50:44,118 INFO [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T09:50:44,118 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731577842392Running coprocessor pre-close hooks at 1731577842393 (+1 ms)Disabling compacts and flushes for region at 1731577842393Disabling writes for close at 1731577842393Obtaining lock to block concurrent updates at 1731577842393Preparing flush snapshotting stores in 1588230740 at 1731577842393Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731577842394 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731577842394Flushing 1588230740/info: creating writer at 1731577842394Flushing 1588230740/info: appending metadata at 1731577842412 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731577842412Flushing 1588230740/ns: creating writer at 1731577842826 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731577842845 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731577842845Flushing 1588230740/rep_barrier: creating writer at 1731577843254 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731577843272 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731577843272Flushing 1588230740/table: creating writer at 1731577843679 (+407 ms)Flushing 1588230740/table: appending metadata at 1731577843692 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731577843692Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65778201: reopening flushed file at 1731577844099 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25865291: reopening flushed file at 1731577844103 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16227698: reopening flushed file at 1731577844107 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@701010dd: reopening flushed file at 1731577844110 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false at 1731577844114 (+4 ms)Writing region close event to WAL at 1731577844115 (+1 ms)Running coprocessor post-close hooks at 1731577844118 (+3 ms)Closed at 1731577844118 2024-11-14T09:50:44,119 DEBUG [RS_CLOSE_META-regionserver/defc576eb6b7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T09:50:44,195 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(976): stopping server defc576eb6b7,40243,1731577658084; all regions closed. 2024-11-14T09:50:44,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741834_1010 (size=25344) 2024-11-14T09:50:44,201 DEBUG [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/oldWALs 2024-11-14T09:50:44,201 INFO [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL defc576eb6b7%2C40243%2C1731577658084.meta:.meta(num 1731577662280) 2024-11-14T09:50:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741832_1008 (size=14749709) 2024-11-14T09:50:44,205 DEBUG [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/oldWALs 2024-11-14T09:50:44,205 INFO [RS:0;defc576eb6b7:40243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL defc576eb6b7%2C40243%2C1731577658084:(num 1731577661240) 2024-11-14T09:50:44,205 DEBUG [RS:0;defc576eb6b7:40243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T09:50:44,205 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T09:50:44,205 INFO [RS:0;defc576eb6b7:40243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:50:44,206 INFO [RS:0;defc576eb6b7:40243 {}] hbase.ChoreService(370): Chore service for: regionserver/defc576eb6b7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T09:50:44,206 INFO [RS:0;defc576eb6b7:40243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:50:44,206 INFO [regionserver/defc576eb6b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:50:44,206 INFO [RS:0;defc576eb6b7:40243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40243 2024-11-14T09:50:44,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/defc576eb6b7,40243,1731577658084 2024-11-14T09:50:44,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T09:50:44,217 INFO [RS:0;defc576eb6b7:40243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:50:44,218 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [defc576eb6b7,40243,1731577658084] 2024-11-14T09:50:44,238 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/defc576eb6b7,40243,1731577658084 already deleted, retry=false 2024-11-14T09:50:44,238 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; defc576eb6b7,40243,1731577658084 expired; onlineServers=0 2024-11-14T09:50:44,238 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'defc576eb6b7,35261,1731577657311' ***** 2024-11-14T09:50:44,239 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T09:50:44,239 DEBUG [M:0;defc576eb6b7:35261 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T09:50:44,239 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T09:50:44,239 DEBUG [M:0;defc576eb6b7:35261 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T09:50:44,239 DEBUG [master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577660971 {}] cleaner.HFileCleaner(306): Exit Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.small.0-1731577660971,5,FailOnTimeoutGroup] 2024-11-14T09:50:44,239 DEBUG [master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577660968 {}] cleaner.HFileCleaner(306): Exit Thread[master/defc576eb6b7:0:becomeActiveMaster-HFileCleaner.large.0-1731577660968,5,FailOnTimeoutGroup] 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] hbase.ChoreService(370): Chore service for: master/defc576eb6b7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T09:50:44,239 DEBUG [M:0;defc576eb6b7:35261 {}] master.HMaster(1795): Stopping service threads 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T09:50:44,239 INFO [M:0;defc576eb6b7:35261 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T09:50:44,240 ERROR [M:0;defc576eb6b7:35261 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T09:50:44,240 INFO [M:0;defc576eb6b7:35261 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T09:50:44,240 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T09:50:44,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T09:50:44,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T09:50:44,249 DEBUG [M:0;defc576eb6b7:35261 {}] zookeeper.ZKUtil(347): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T09:50:44,249 WARN [M:0;defc576eb6b7:35261 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T09:50:44,249 INFO [M:0;defc576eb6b7:35261 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/.lastflushedseqids 2024-11-14T09:50:44,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742478_1654 (size=72) 2024-11-14T09:50:44,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:50:44,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40243-0x10138be301c0001, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:50:44,328 INFO [RS:0;defc576eb6b7:40243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:50:44,329 INFO [RS:0;defc576eb6b7:40243 {}] regionserver.HRegionServer(1031): Exiting; stopping=defc576eb6b7,40243,1731577658084; zookeeper connection closed. 2024-11-14T09:50:44,329 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28c97637 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28c97637 2024-11-14T09:50:44,330 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T09:50:44,658 INFO [M:0;defc576eb6b7:35261 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T09:50:44,659 INFO [M:0;defc576eb6b7:35261 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T09:50:44,659 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T09:50:44,659 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:50:44,659 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:50:44,659 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T09:50:44,659 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:50:44,659 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=787.18 KB heapSize=970.11 KB 2024-11-14T09:50:44,677 DEBUG [M:0;defc576eb6b7:35261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/952db54c79c146c0ad64b7c93b419b6b is 82, key is hbase:meta,,1/info:regioninfo/1731577662369/Put/seqid=0 2024-11-14T09:50:44,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742479_1655 (size=5672) 2024-11-14T09:50:45,081 INFO [M:0;defc576eb6b7:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/952db54c79c146c0ad64b7c93b419b6b 2024-11-14T09:50:45,109 DEBUG [M:0;defc576eb6b7:35261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c42dcf0497604bc3a46988125dc75f84 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731577812844/Put/seqid=0 2024-11-14T09:50:45,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742480_1656 (size=44887) 2024-11-14T09:50:45,520 INFO [M:0;defc576eb6b7:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=786.63 KB at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c42dcf0497604bc3a46988125dc75f84 2024-11-14T09:50:45,523 INFO [M:0;defc576eb6b7:35261 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c42dcf0497604bc3a46988125dc75f84 2024-11-14T09:50:45,537 DEBUG [M:0;defc576eb6b7:35261 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/321fb01a0a934ee8a5da7b51ecb10a6d is 69, key is defc576eb6b7,40243,1731577658084/rs:state/1731577661002/Put/seqid=0 2024-11-14T09:50:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073742481_1657 (size=5156) 2024-11-14T09:50:45,941 INFO [M:0;defc576eb6b7:35261 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2265 (bloomFilter=true), to=hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/321fb01a0a934ee8a5da7b51ecb10a6d 2024-11-14T09:50:45,945 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/952db54c79c146c0ad64b7c93b419b6b as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/952db54c79c146c0ad64b7c93b419b6b 2024-11-14T09:50:45,947 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/952db54c79c146c0ad64b7c93b419b6b, entries=8, sequenceid=2265, filesize=5.5 K 2024-11-14T09:50:45,948 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c42dcf0497604bc3a46988125dc75f84 as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c42dcf0497604bc3a46988125dc75f84 2024-11-14T09:50:45,950 INFO [M:0;defc576eb6b7:35261 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c42dcf0497604bc3a46988125dc75f84 2024-11-14T09:50:45,950 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c42dcf0497604bc3a46988125dc75f84, entries=174, sequenceid=2265, filesize=43.8 K 2024-11-14T09:50:45,951 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/321fb01a0a934ee8a5da7b51ecb10a6d as hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/321fb01a0a934ee8a5da7b51ecb10a6d 2024-11-14T09:50:45,954 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44807/user/jenkins/test-data/e1e0ec74-6609-5100-1de0-9fc49d77e9f3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/321fb01a0a934ee8a5da7b51ecb10a6d, entries=1, sequenceid=2265, filesize=5.0 K 2024-11-14T09:50:45,955 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(3140): Finished flush of dataSize ~787.18 KB/806075, heapSize ~969.81 KB/993088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2265, compaction requested=false 2024-11-14T09:50:45,957 INFO [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T09:50:45,957 DEBUG [M:0;defc576eb6b7:35261 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731577844659Disabling compacts and flushes for region at 1731577844659Disabling writes for close at 1731577844659Obtaining lock to block concurrent updates at 1731577844659Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731577844659Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=806075, getHeapSize=993328, getOffHeapSize=0, getCellsCount=2464 at 1731577844660 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731577844660Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731577844660Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731577844677 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731577844677Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731577845084 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731577845109 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731577845109Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731577845523 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731577845537 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731577845537Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75847c88: reopening flushed file at 1731577845944 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6caea153: reopening flushed file at 1731577845947 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ba4d8a: reopening flushed file at 1731577845950 (+3 ms)Finished flush of dataSize ~787.18 KB/806075, heapSize ~969.81 KB/993088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2265, compaction requested=false at 1731577845955 (+5 ms)Writing region close event to WAL at 1731577845957 (+2 ms)Closed at 1731577845957 2024-11-14T09:50:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44663 is added to blk_1073741830_1006 (size=954515) 2024-11-14T09:50:45,960 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T09:50:45,960 INFO [M:0;defc576eb6b7:35261 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T09:50:45,961 INFO [M:0;defc576eb6b7:35261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35261 2024-11-14T09:50:45,961 INFO [M:0;defc576eb6b7:35261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T09:50:46,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:50:46,075 INFO [M:0;defc576eb6b7:35261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T09:50:46,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35261-0x10138be301c0000, quorum=127.0.0.1:52200, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T09:50:46,083 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T09:50:46,087 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:50:46,087 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:50:46,087 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:50:46,088 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.log.dir/,STOPPED} 2024-11-14T09:50:46,091 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T09:50:46,091 WARN [BP-1058783131-172.17.0.2-1731577653437 heartbeating to localhost/127.0.0.1:44807 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T09:50:46,091 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T09:50:46,091 WARN [BP-1058783131-172.17.0.2-1731577653437 heartbeating to localhost/127.0.0.1:44807 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1058783131-172.17.0.2-1731577653437 (Datanode Uuid c8d1fb9d-fa1d-42ec-9043-d1b24d0eb8ba) service to localhost/127.0.0.1:44807 2024-11-14T09:50:46,093 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/data/data1/current/BP-1058783131-172.17.0.2-1731577653437 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:50:46,093 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/cluster_258991e0-3b54-b676-d819-991f7f4f5c6b/data/data2/current/BP-1058783131-172.17.0.2-1731577653437 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T09:50:46,094 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T09:50:46,100 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T09:50:46,100 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T09:50:46,100 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T09:50:46,100 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T09:50:46,101 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5f19eaed-47ee-3c3c-9cda-c2d658f5bdd6/hadoop.log.dir/,STOPPED} 2024-11-14T09:50:46,115 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T09:50:46,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down